first commit

packages/queue/README.md

@@ -0,0 +1,164 @@
+# @wrnexus/queue
+
+> A background job queue with delays, retries + exponential backoff, recurring jobs, and concurrent workers.
+
+Part of the **WrNexus** framework — an SSR-first, Bun-native full-stack web framework.
+
+## Overview
+
+`@wrnexus/queue` is a server-side in-process job queue. You register named
+workers, enqueue jobs (optionally delayed or recurring), and let the queue poll
+and run them on a timer — with per-job retry limits and doubling backoff between
+attempts. The default store lives in memory; the design allows a pluggable driver
+to back it with Redis/SQL for durability across restarts. Reach for it when you
+need to defer work (emails, webhooks, cleanup) off the request path without a
+heavyweight external broker. Tests can drive it deterministically via `drain()`.
+
+## Installation
+
+```bash
+bun add @wrnexus/queue
+```
+
+> Private package — the machine must be authenticated to the `wrnexus` npm org
+> (a read token in `~/.npmrc`). Requires **Bun** (Node is not supported).
+
+## API
+
+The package exports a single factory plus its supporting types.
+
+### `createQueue(options?): Queue`
+
+Creates a new queue instance.
+
+```ts
+function createQueue(options?: QueueOptions): Queue;
+```
+
+#### `QueueOptions`
+
+| Option        | Type                                 | Default    | Description                                              |
+| ------------- | ------------------------------------ | ---------- | -------------------------------------------------------- |
+| `maxAttempts` | `number`                             | `3`        | Default max attempts per job before it is dead-lettered. |
+| `backoffMs`   | `number`                             | `1000`     | Base retry backoff in ms; doubles per attempt.           |
+| `pollMs`      | `number`                             | `250`      | Poll interval used once `start()` is called (ms).        |
+| `onFailed`    | `(job: Job, error: unknown) => void` | —          | Called when a job exhausts its attempts.                 |
+| `now`         | `() => number`                       | `Date.now` | Clock injection for deterministic tests.                 |
+
+### `Queue`
+
+The object returned by `createQueue`.
+
+| Method    | Signature                                                      | Description                                                    |
+| --------- | -------------------------------------------------------------- | -------------------------------------------------------------- |
+| `add`     | `add<T>(name, data: T, options?: AddOptions): Promise<Job<T>>` | Enqueue a job under a worker name. Returns the created job.    |
+| `process` | `process<T>(name, handler: JobHandler<T>): void`               | Register the worker that runs jobs of the given name.          |
+| `drain`   | `drain(now?: number): Promise<number>`                         | Run every job whose `runAt ≤ now`, once. Returns how many ran. |
+| `start`   | `start(): void`                                                | Begin polling every `pollMs`. No-op if already started.        |
+| `stop`    | `stop(): void`                                                 | Stop the poll timer.                                           |
+| `size`    | `size(): number`                                               | Number of jobs currently queued.                               |
+
+#### `AddOptions`
+
+| Option        | Type     | Description                                                                |
+| ------------- | -------- | -------------------------------------------------------------------------- |
+| `delayMs`     | `number` | Delay before the job becomes runnable (ms).                                |
+| `maxAttempts` | `number` | Max attempts before dead-lettering. Defaults to the queue's `maxAttempts`. |
+| `repeat`      | `number` | Re-enqueue this job this many ms after each successful run (recurring).    |
+
+#### `JobHandler<T>`
+
+```ts
+type JobHandler<T = unknown> = (job: Job<T>) => void | Promise<void>;
+```
+
+#### `Job<T>`
+
+```ts
+interface Job<T = unknown> {
+  id: string; // e.g. "job_1"
+  name: string;
+  data: T;
+  attempts: number;
+  maxAttempts: number;
+  runAt: number; // epoch ms; job runs when now ≥ runAt
+  repeat?: number; // if set, re-enqueue this many ms after each success
+}
+```
+
+## Usage
+
+Register workers, enqueue jobs, then start the poller:
+
+```ts
+import { createQueue } from "@wrnexus/queue";
+
+const queue = createQueue({ maxAttempts: 3, backoffMs: 1000 });
+
+// Register a worker for the "email" job name.
+queue.process<{ to: string }>("email", async (job) => {
+  await send(job.data.to);
+});
+
+// Enqueue a delayed job with up to 3 attempts.
+await queue.add("email", { to: "a@b.com" }, { delayMs: 5000, maxAttempts: 3 });
+
+queue.start(); // begin polling; queue.stop() to halt
+```
+
+### Recurring jobs
+
+Pass `repeat` to re-enqueue a job a fixed interval after each successful run:
+
+```ts
+queue.process("heartbeat", async () => ping());
+await queue.add("heartbeat", {}, { repeat: 60_000 }); // runs ~every minute
+```
+
+### Handling permanent failures
+
+When a job's `attempts` reaches `maxAttempts`, it is dropped and `onFailed`
+fires instead of retrying:
+
+```ts
+const queue = createQueue({
+  onFailed: (job, error) => {
+    console.error(`job ${job.id} (${job.name}) gave up`, error);
+  },
+});
+```
+
+### Deterministic testing
+
+Instead of `start()`, inject a clock and drive the queue with `drain()`:
+
+```ts
+let clock = 0;
+const queue = createQueue({ now: () => clock });
+
+queue.process("task", async () => {
+  /* ... */
+});
+await queue.add("task", {}, { delayMs: 5000 });
+
+clock = 5000;
+const ran = await queue.drain(); // => 1
+```
+
+## Retry & backoff behavior
+
+- On a thrown handler error, the job is retried while `attempts < maxAttempts`.
+- The next `runAt` is set to `now + backoffMs * 2^(attempts - 1)` (exponential
+  backoff): with `backoffMs: 1000` the delays are 1s, 2s, 4s, …
+- A job whose worker name has no registered handler stays queued until one is
+  registered (it is not counted as runnable by `drain`).
+- `drain` is re-entrant-safe: overlapping calls are skipped while one is running.
+
+## Requirements / Notes
+
+- **Bun-only** runtime (Node is not supported), consistent with the rest of the
+  WrNexus framework. The queue itself relies only on standard timers
+  (`setInterval`/`clearInterval`) and has no runtime dependencies.
+- The default store is in-process, so queued jobs do not survive a restart; a
+  pluggable driver is intended for backing it with Redis/SQL for durability.
+- Works alongside `@wrnexus/core` for offloading work from the request path.

packages/queue/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@wrnexus/queue",
+  "version": "0.2.12",
+  "private": true,
+  "type": "module",
+  "main": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  }
+}

packages/queue/src/index.ts

@@ -0,0 +1,149 @@
+/**
+ * @wrnexus/queue — a background job queue with delays, retries + backoff, and
+ * concurrent workers. The default store is in-process; a pluggable driver lets
+ * you back it with Redis/SQL for durability across restarts.
+ *
+ *   const queue = createQueue();
+ *   queue.process("email", async (job) => { await send(job.data); });
+ *   await queue.add("email", { to: "a@b.com" }, { delayMs: 5000, maxAttempts: 3 });
+ *   queue.start();                 // begin polling; queue.stop() to halt
+ *
+ * Tests can drive it deterministically with `await queue.drain(now)`.
+ */
+
+export interface Job<T = unknown> {
+  id: string;
+  name: string;
+  data: T;
+  attempts: number;
+  maxAttempts: number;
+  runAt: number;
+  /** If set, re-enqueue this job this many ms after each successful run. */
+  repeat?: number;
+}
+
+export type JobHandler<T = unknown> = (job: Job<T>) => void | Promise<void>;
+
+export interface AddOptions {
+  /** Delay before the job becomes runnable (ms). */
+  delayMs?: number;
+  /** Max attempts before it's dead-lettered. Default from queue options. */
+  maxAttempts?: number;
+  /** Re-enqueue this job this many ms after each successful run (recurring). */
+  repeat?: number;
+}
+
+export interface QueueOptions {
+  /** Default max attempts per job. Default 3. */
+  maxAttempts?: number;
+  /** Base retry backoff (ms); doubles per attempt. Default 1000. */
+  backoffMs?: number;
+  /** Poll interval when started (ms). Default 250. */
+  pollMs?: number;
+  /** Called when a job exhausts its attempts. */
+  onFailed?: (job: Job, error: unknown) => void;
+  /** Clock injection (tests). Default Date.now. */
+  now?: () => number;
+}
+
+export interface Queue {
+  add<T>(name: string, data: T, options?: AddOptions): Promise<Job<T>>;
+  process<T>(name: string, handler: JobHandler<T>): void;
+  /** Run every job whose runAt ≤ now, once. Returns how many ran. */
+  drain(now?: number): Promise<number>;
+  start(): void;
+  stop(): void;
+  size(): number;
+}
+
+export function createQueue(options: QueueOptions = {}): Queue {
+  const defaultMax = options.maxAttempts ?? 3;
+  const backoffMs = options.backoffMs ?? 1000;
+  const pollMs = options.pollMs ?? 250;
+  if (!Number.isInteger(defaultMax) || defaultMax < 1)
+    throw new RangeError("queue maxAttempts must be a positive integer");
+  if (!Number.isFinite(backoffMs) || backoffMs < 0)
+    throw new RangeError("queue backoffMs must be a non-negative number");
+  if (!Number.isFinite(pollMs) || pollMs < 1)
+    throw new RangeError("queue pollMs must be at least 1ms");
+  const now = options.now ?? Date.now;
+
+  const jobs: Job[] = [];
+  const handlers = new Map<string, JobHandler>();
+  let seq = 0;
+  let timer: ReturnType<typeof setInterval> | null = null;
+  let draining = false;
+
+  async function runJob(job: Job): Promise<void> {
+    const handler = handlers.get(job.name);
+    if (!handler) return; // no worker registered yet — leave it queued
+    const idx = jobs.indexOf(job);
+    if (idx >= 0) jobs.splice(idx, 1); // claim it
+    job.attempts++;
+    try {
+      await handler(job);
+      if (job.repeat && job.repeat > 0) {
+        jobs.push({ ...job, attempts: 0, runAt: now() + job.repeat }); // recurring
+      }
+    } catch (error) {
+      if (job.attempts < job.maxAttempts) {
+        job.runAt = now() + backoffMs * Math.pow(2, job.attempts - 1); // exponential backoff
+        jobs.push(job);
+      } else {
+        options.onFailed?.(job, error);
+      }
+    }
+  }
+
+  const drain: Queue["drain"] = async (at) => {
+    if (draining) return 0;
+    draining = true;
+    try {
+      const cutoff = at ?? now();
+      const due = jobs.filter((j) => j.runAt <= cutoff && handlers.has(j.name));
+      await Promise.all(due.map(runJob));
+      return due.length;
+    } finally {
+      draining = false;
+    }
+  };
+
+  return {
+    async add(name, data, opts = {}) {
+      if (!name.trim()) throw new TypeError("queue job name cannot be empty");
+      if (
+        opts.maxAttempts !== undefined &&
+        (!Number.isInteger(opts.maxAttempts) || opts.maxAttempts < 1)
+      )
+        throw new RangeError("job maxAttempts must be a positive integer");
+      if (opts.delayMs !== undefined && (!Number.isFinite(opts.delayMs) || opts.delayMs < 0))
+        throw new RangeError("job delayMs must be a non-negative number");
+      if (opts.repeat !== undefined && (!Number.isFinite(opts.repeat) || opts.repeat <= 0))
+        throw new RangeError("job repeat must be a positive number");
+      const job: Job = {
+        id: `job_${++seq}`,
+        name,
+        data,
+        attempts: 0,
+        maxAttempts: opts.maxAttempts ?? defaultMax,
+        runAt: now() + (opts.delayMs ?? 0),
+        repeat: opts.repeat,
+      };
+      jobs.push(job);
+      return job as Job<typeof data>;
+    },
+    process(name, handler) {
+      handlers.set(name, handler as JobHandler);
+    },
+    drain,
+    start() {
+      if (timer) return;
+      timer = setInterval(() => void drain(), pollMs);
+    },
+    stop() {
+      if (timer) clearInterval(timer);
+      timer = null;
+    },
+    size: () => jobs.length,
+  };
+}

packages/queue/test/queue.test.ts

@@ -0,0 +1,106 @@
+import { test, expect } from "bun:test";
+import { createQueue } from "../src/index.ts";
+
+test("processes a job", async () => {
+  const queue = createQueue();
+  const done: string[] = [];
+  queue.process<{ to: string }>("email", (job) => {
+    done.push(job.data.to);
+  });
+  await queue.add("email", { to: "a@b.com" });
+  expect(queue.size()).toBe(1);
+  await queue.drain();
+  expect(done).toEqual(["a@b.com"]);
+  expect(queue.size()).toBe(0);
+});
+
+test("delayed jobs only run once due", async () => {
+  let clock = 1000;
+  const queue = createQueue({ now: () => clock });
+  const ran: number[] = [];
+  queue.process("x", () => {
+    ran.push(clock);
+  });
+  await queue.add("x", 1, { delayMs: 500 });
+  await queue.drain(1200); // not yet due (runAt = 1500)
+  expect(ran.length).toBe(0);
+  clock = 1600;
+  await queue.drain();
+  expect(ran.length).toBe(1);
+});
+
+test("retries with backoff, then dead-letters", async () => {
+  let clock = 0;
+  const failed: unknown[] = [];
+  const queue = createQueue({
+    maxAttempts: 3,
+    backoffMs: 100,
+    now: () => clock,
+    onFailed: (job) => failed.push(job.id),
+  });
+  let calls = 0;
+  queue.process("flaky", () => {
+    calls++;
+    throw new Error("boom");
+  });
+  await queue.add("flaky", {});
+  await queue.drain(); // attempt 1 → reschedule at 100
+  expect(calls).toBe(1);
+  clock = 100;
+  await queue.drain(); // attempt 2 → reschedule at 0+200
+  clock = 300;
+  await queue.drain(); // attempt 3 → dead-letter
+  expect(calls).toBe(3);
+  expect(failed.length).toBe(1);
+  expect(queue.size()).toBe(0);
+});
+
+test("repeat re-enqueues a recurring job after each success", async () => {
+  let clock = 0;
+  const queue = createQueue({ now: () => clock });
+  let runs = 0;
+  queue.process("tick", () => {
+    runs++;
+  });
+  await queue.add("tick", {}, { repeat: 1000 });
+  await queue.drain(); // run 1 → re-enqueued at 1000
+  expect(runs).toBe(1);
+  await queue.drain(); // not due yet
+  expect(runs).toBe(1);
+  clock = 1000;
+  await queue.drain(); // run 2 → re-enqueued at 2000
+  expect(runs).toBe(2);
+  expect(queue.size()).toBe(1); // always one pending
+});
+
+test("jobs without a worker stay queued", async () => {
+  const queue = createQueue();
+  await queue.add("later", {});
+  await queue.drain();
+  expect(queue.size()).toBe(1); // no handler yet
+  queue.process("later", () => {});
+  await queue.drain();
+  expect(queue.size()).toBe(0);
+});
+
+test("rejects invalid queue and job timing options", async () => {
+  expect(() => createQueue({ pollMs: 0 })).toThrow("pollMs");
+  expect(() => createQueue({ maxAttempts: 0 })).toThrow("maxAttempts");
+  const queue = createQueue();
+  await expect(queue.add("", {})).rejects.toThrow("name");
+  await expect(queue.add("job", {}, { delayMs: -1 })).rejects.toThrow("delayMs");
+  await expect(queue.add("job", {}, { repeat: 0 })).rejects.toThrow("repeat");
+});
+
+test("drains independent due jobs concurrently", async () => {
+  const queue = createQueue();
+  const releases: Array<() => void> = [];
+  queue.process("work", () => new Promise<void>((resolve) => releases.push(resolve)));
+  await queue.add("work", 1);
+  await queue.add("work", 2);
+  const draining = queue.drain();
+  await Promise.resolve();
+  expect(releases).toHaveLength(2);
+  releases.forEach((release) => release());
+  expect(await draining).toBe(2);
+});