feat: add application productivity foundations

packages/queue/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wrnexus/queue",
-  "version": "0.8.12",
+  "version": "0.8.13",
   "private": true,
   "type": "module",
   "main": "src/index.ts",

packages/queue/src/admin.ts

@@ -0,0 +1,37 @@
+import type { DurableQueue } from "./durable.ts";
+import { queueDashboardSnapshot, renderQueueDashboard } from "./scheduler.ts";
+
+export interface QueueAdminOptions {
+  queues: Record<string, DurableQueue>;
+  authorize?: (request: Request) => boolean | Promise<boolean>;
+}
+
+/** Framework-neutral protected queue dashboard/API handler, mountable at any route. */
+export function createQueueAdminHandler(options: QueueAdminOptions) {
+  return async (request: Request): Promise<Response> => {
+    if (options.authorize && !(await options.authorize(request))) {
+      return Response.json({ error: "Forbidden" }, { status: 403 });
+    }
+    const url = new URL(request.url);
+    const name = url.searchParams.get("queue") ?? Object.keys(options.queues)[0];
+    const queue = name ? options.queues[name] : undefined;
+    if (!queue) return Response.json({ error: "Queue not found" }, { status: 404 });
+    const id = url.searchParams.get("id");
+    if (request.method === "POST" && id) {
+      const action = url.searchParams.get("action");
+      const changed =
+        action === "retry"
+          ? await queue.retry(id)
+          : action === "cancel"
+            ? await queue.cancel(id)
+            : false;
+      return Response.json({ ok: changed });
+    }
+    const snapshot = await queueDashboardSnapshot(queue);
+    if (url.searchParams.get("format") === "json")
+      return Response.json({ queue: name, ...snapshot });
+    return new Response(renderQueueDashboard(snapshot), {
+      headers: { "content-type": "text/html; charset=utf-8" },
+    });
+  };
+}

packages/queue/src/configured.ts

@@ -2,6 +2,7 @@ import { getDb, hasDb, registerDb, type Db } from "@wrnexus/db";
 import { connectFromConfig } from "@wrnexus/db/connect";
 import { memoryQueueStore, type QueueStore } from "./durable.ts";
 import { installSqliteQueueSchema, sqliteQueueStore } from "./sqlite.ts";
+import { databaseQueueStore, installDatabaseQueueSchema } from "./database.ts";
 
 export type QueueStorage = "sqlite" | "database" | "memory";
 
@@ -29,21 +30,39 @@ function lazyStore(resolve: () => Promise<QueueStore>): QueueStore {
   let pending: Promise<QueueStore> | undefined;
   const ready = () => (pending ??= resolve());
   return {
-    async put(job) { return (await ready()).put(job); },
-    async get(id) { return (await ready()).get(id); },
-    async remove(id) { return (await ready()).remove(id); },
-    async due(now, limit) { return (await ready()).due(now, limit); },
-    async list(name) { return (await ready()).list(name); },
+    async put(job) {
+      return (await ready()).put(job);
+    },
+    async get(id) {
+      return (await ready()).get(id);
+    },
+    async remove(id) {
+      return (await ready()).remove(id);
+    },
+    async due(now, limit) {
+      return (await ready()).due(now, limit);
+    },
+    async list(name) {
+      return (await ready()).list(name);
+    },
     async findByIdempotencyKey(name, key) {
       return (await ready()).findByIdempotencyKey?.(name, key) ?? null;
     },
-    async size() { return (await ready()).size?.() ?? (await (await ready()).list()).length; },
+    async size() {
+      return (await ready()).size?.() ?? (await (await ready()).list()).length;
+    },
     async claim(id, worker, leaseUntil, now) {
       return (await ready()).claim?.(id, worker, leaseUntil, now) ?? true;
     },
-    async release(id, worker) { await (await ready()).release?.(id, worker); },
-    async archive(record) { await (await ready()).archive?.(record); },
-    async history(id) { return (await ready()).history?.(id) ?? []; },
+    async release(id, worker) {
+      await (await ready()).release?.(id, worker);
+    },
+    async archive(record) {
+      await (await ready()).archive?.(record);
+    },
+    async history(id) {
+      return (await ready()).history?.(id) ?? [];
+    },
   };
 }
 
@@ -72,15 +91,13 @@ async function resolveConfiguredStore(): Promise<QueueStore> {
     }
   }
 
-  if (db.driver.dialect !== "sqlite") {
-    throw new Error(
-      `WRN-QUEUE-DATABASE: configured queue storage currently requires SQLite; ` +
-        `database '${databaseName}' uses ${db.driver.dialect}. Pass a custom queue store for that driver.`,
-    );
-  }
   const table = config.table ?? "wrnexus_jobs";
-  await installSqliteQueueSchema(db, table);
-  return sqliteQueueStore(db, table);
+  if (db.driver.dialect === "sqlite") {
+    await installSqliteQueueSchema(db, table);
+    return sqliteQueueStore(db, table);
+  }
+  await installDatabaseQueueSchema(db, table);
+  return databaseQueueStore(db, table);
 }
 
 /** Lazy global store used by defineQueue; runtime config is read on first operation. */

packages/queue/src/database.ts

@@ -0,0 +1,158 @@
+import type { Db } from "@wrnexus/db";
+import type { Job } from "./index.ts";
+import type { QueueJobRecord, QueueStore } from "./durable.ts";
+
+const SAFE = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const check = (value: string) => {
+  if (!SAFE.test(value)) throw new TypeError(`Invalid queue table: ${value}`);
+  return value;
+};
+
+export async function installDatabaseQueueSchema(db: Db, table = "wrnexus_jobs") {
+  const target = check(table);
+  await db.exec(`CREATE TABLE IF NOT EXISTS ${target} (
+    id VARCHAR(191) PRIMARY KEY, name VARCHAR(191) NOT NULL, payload TEXT NOT NULL,
+    run_at BIGINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0,
+    idempotency_key VARCHAR(191), lease_owner VARCHAR(191), lease_until BIGINT
+  )`);
+  await db.exec(`CREATE TABLE IF NOT EXISTS ${target}_history (
+    id VARCHAR(191) PRIMARY KEY, name VARCHAR(191) NOT NULL, state VARCHAR(32) NOT NULL,
+    payload TEXT NOT NULL, error TEXT, finished_at BIGINT NOT NULL
+  )`);
+  for (const statement of [
+    `CREATE INDEX ${target}_due ON ${target} (run_at, priority)`,
+    `CREATE UNIQUE INDEX ${target}_idempotency ON ${target} (name, idempotency_key)`,
+  ]) {
+    try {
+      await db.exec(statement);
+    } catch (error) {
+      if (!/exist|duplicate|already/i.test(String(error))) throw error;
+    }
+  }
+}
+
+export function databaseQueueStore(db: Db, table = "wrnexus_jobs"): QueueStore {
+  const target = check(table);
+  type Row = { payload: string };
+  const decode = (row: Row) => JSON.parse(row.payload) as Job;
+  return {
+    async put(job) {
+      if (db.driver.dialect === "mysql") {
+        await db.exec(
+          `INSERT INTO ${target} (id,name,payload,run_at,priority,idempotency_key,lease_owner,lease_until)
+          VALUES (?,?,?,?,?,?,NULL,NULL) ON DUPLICATE KEY UPDATE name=VALUES(name),payload=VALUES(payload),run_at=VALUES(run_at),priority=VALUES(priority),lease_owner=NULL,lease_until=NULL`,
+          [
+            job.id,
+            job.name,
+            JSON.stringify(job),
+            job.runAt,
+            job.priority,
+            job.idempotencyKey ?? null,
+          ],
+        );
+      } else {
+        await db.exec(
+          `INSERT INTO ${target} (id,name,payload,run_at,priority,idempotency_key,lease_owner,lease_until)
+          VALUES (?,?,?,?,?,?,NULL,NULL) ON CONFLICT(id) DO UPDATE SET name=excluded.name,payload=excluded.payload,run_at=excluded.run_at,priority=excluded.priority,lease_owner=NULL,lease_until=NULL`,
+          [
+            job.id,
+            job.name,
+            JSON.stringify(job),
+            job.runAt,
+            job.priority,
+            job.idempotencyKey ?? null,
+          ],
+        );
+      }
+    },
+    async get(id) {
+      const row = await db.one<Row>(`SELECT payload FROM ${target} WHERE id = ?`, [id]);
+      return row ? decode(row) : null;
+    },
+    async remove(id) {
+      await db.exec(`DELETE FROM ${target} WHERE id = ?`, [id]);
+    },
+    async due(now, limit) {
+      return (
+        await db.all<Row>(
+          `SELECT payload FROM ${target} WHERE run_at <= ? AND (lease_until IS NULL OR lease_until < ?) ORDER BY priority DESC,run_at LIMIT ?`,
+          [now, now, limit],
+        )
+      ).map(decode);
+    },
+    async list(name) {
+      return (
+        await db.all<Row>(
+          `SELECT payload FROM ${target}${name ? " WHERE name = ?" : ""} ORDER BY run_at`,
+          name ? [name] : [],
+        )
+      ).map(decode);
+    },
+    async findByIdempotencyKey(name, key) {
+      const row = await db.one<Row>(
+        `SELECT payload FROM ${target} WHERE name = ? AND idempotency_key = ?`,
+        [name, key],
+      );
+      return row ? decode(row) : null;
+    },
+    async size() {
+      return Number(
+        (await db.one<{ total: number | string }>(`SELECT COUNT(*) AS total FROM ${target}`))
+          ?.total ?? 0,
+      );
+    },
+    async claim(id, worker, leaseUntil, now = Date.now()) {
+      return (
+        (
+          await db.exec(
+            `UPDATE ${target} SET lease_owner=?,lease_until=? WHERE id=? AND (lease_until IS NULL OR lease_until < ?)`,
+            [worker, leaseUntil, id, now],
+          )
+        ).changes > 0
+      );
+    },
+    async release(id, worker) {
+      await db.exec(
+        `UPDATE ${target} SET lease_owner=NULL,lease_until=NULL WHERE id=? AND lease_owner=?`,
+        [id, worker],
+      );
+    },
+    async archive(record) {
+      const values = [
+        record.job.id,
+        record.job.name,
+        record.state,
+        JSON.stringify(record.job),
+        record.error ?? null,
+        record.finishedAt,
+      ];
+      if (db.driver.dialect === "mysql")
+        await db.exec(
+          `INSERT INTO ${target}_history (id,name,state,payload,error,finished_at) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE state=VALUES(state),payload=VALUES(payload),error=VALUES(error),finished_at=VALUES(finished_at)`,
+          values,
+        );
+      else
+        await db.exec(
+          `INSERT INTO ${target}_history (id,name,state,payload,error,finished_at) VALUES (?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET state=excluded.state,payload=excluded.payload,error=excluded.error,finished_at=excluded.finished_at`,
+          values,
+        );
+    },
+    async history(id) {
+      const rows = await db.all<{
+        state: QueueJobRecord["state"];
+        payload: string;
+        error: string | null;
+        finished_at: number;
+      }>(
+        `SELECT state,payload,error,finished_at FROM ${target}_history${id ? " WHERE id = ?" : ""} ORDER BY finished_at DESC`,
+        id ? [id] : [],
+      );
+      return rows.map((row) => ({
+        job: JSON.parse(row.payload),
+        state: row.state,
+        error: row.error ?? undefined,
+        finishedAt: Number(row.finished_at),
+      }));
+    },
+  };
+}

packages/queue/src/index.ts

@@ -317,6 +317,7 @@ export type {
 } from "./defined.ts";
 export { installSqliteQueueSchema, sqliteQueueSchema, sqliteQueueStore } from "./sqlite.ts";
 export type { SqliteQueueClient } from "./sqlite.ts";
+export { databaseQueueStore, installDatabaseQueueSchema } from "./database.ts";
 export { configureQueueStorage, configuredQueueStore, queueStorageConfig } from "./configured.ts";
 export type { QueueStorage, QueueStorageConfig } from "./configured.ts";
 export { redisQueueStore, postgresQueueStore, POSTGRES_QUEUE_SCHEMA } from "./stores.ts";
@@ -340,4 +341,15 @@ export type {
 } from "./workflow.ts";
 export { subjectQueue } from "./subject.ts";
 export type { SubjectJob, SubjectQueue } from "./subject.ts";
+export {
+  defineJobStateMachine,
+  defineSaga,
+  drainOutbox,
+  installOutboxSchema,
+  testQueue,
+  withOutbox,
+} from "./patterns.ts";
+export { createQueueAdminHandler } from "./admin.ts";
+export type { QueueAdminOptions } from "./admin.ts";
+export type { JobStateMachineOptions, OutboxEntry, OutboxWriter, SagaStep } from "./patterns.ts";
 import { createExecutionContext, type ExecutionContext } from "@wrnexus/core";

packages/queue/src/patterns.ts

@@ -0,0 +1,171 @@
+import type { Db } from "@wrnexus/db";
+import type { AddOptions, Job } from "./index.ts";
+import type { DefinedJob } from "./defined.ts";
+import type { DurableQueue } from "./durable.ts";
+
+const SAFE = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const ident = (value: string) => {
+  if (!SAFE.test(value)) throw new TypeError(`Unsafe queue identifier: ${value}`);
+  return value;
+};
+
+export interface OutboxEntry<T = unknown> {
+  id: number;
+  queue: string;
+  payload: T;
+  options?: AddOptions;
+}
+
+export interface OutboxWriter {
+  enqueue<T>(job: Pick<DefinedJob<T>, "name">, data: T, options?: AddOptions): Promise<void>;
+}
+
+export async function installOutboxSchema(db: Db, table = "wrnexus_outbox"): Promise<void> {
+  const target = ident(table);
+  const id =
+    db.driver.dialect === "postgres"
+      ? "BIGSERIAL PRIMARY KEY"
+      : "INTEGER PRIMARY KEY AUTOINCREMENT";
+  await db.exec(`CREATE TABLE IF NOT EXISTS ${target} (
+    id ${id}, queue_name TEXT NOT NULL, payload TEXT NOT NULL,
+    options TEXT, created_at BIGINT NOT NULL, dispatched_at BIGINT
+  )`);
+  await db.exec(`CREATE INDEX IF NOT EXISTS ${target}_pending ON ${target} (dispatched_at, id)`);
+}
+
+/** Commit business writes and durable enqueue intents in the same database transaction. */
+export async function withOutbox<T>(
+  db: Db,
+  operation: (context: { db: Db; enqueue: OutboxWriter["enqueue"] }) => Promise<T>,
+  table = "wrnexus_outbox",
+): Promise<T> {
+  await installOutboxSchema(db, table);
+  const target = ident(table);
+  return db.tx((transaction) =>
+    operation({
+      db: transaction,
+      enqueue: async (job, data, options) => {
+        await transaction.exec(
+          `INSERT INTO ${target} (queue_name,payload,options,created_at,dispatched_at) VALUES (?,?,?,?,NULL)`,
+          [job.name, JSON.stringify(data), options ? JSON.stringify(options) : null, Date.now()],
+        );
+      },
+    }),
+  );
+}
+
+/** Dispatch committed outbox rows. A crash before marking simply retries idempotently. */
+export async function drainOutbox(
+  db: Db,
+  resolve: (
+    queueName: string,
+  ) => { add(data: unknown, options?: AddOptions): Promise<Job> } | undefined,
+  options: { table?: string; limit?: number } = {},
+): Promise<number> {
+  const target = ident(options.table ?? "wrnexus_outbox");
+  await installOutboxSchema(db, target);
+  const rows = await db.all<{
+    id: number;
+    queue_name: string;
+    payload: string;
+    options: string | null;
+  }>(
+    `SELECT id,queue_name,payload,options FROM ${target} WHERE dispatched_at IS NULL ORDER BY id LIMIT ?`,
+    [options.limit ?? 100],
+  );
+  let dispatched = 0;
+  for (const row of rows) {
+    const producer = resolve(row.queue_name);
+    if (!producer) continue;
+    const addOptions = row.options ? (JSON.parse(row.options) as AddOptions) : {};
+    await producer.add(JSON.parse(row.payload), {
+      idempotencyKey: `outbox:${row.id}`,
+      ...addOptions,
+    });
+    await db.exec(`UPDATE ${target} SET dispatched_at = ? WHERE id = ? AND dispatched_at IS NULL`, [
+      Date.now(),
+      row.id,
+    ]);
+    dispatched++;
+  }
+  return dispatched;
+}
+
+export interface JobStateMachineOptions {
+  db: () => Db;
+  table: string;
+  idColumn?: string;
+  stateColumn?: string;
+  states: { queued: string; running: string; success: string; failed: string };
+}
+
+export function defineJobStateMachine(options: JobStateMachineOptions) {
+  const table = ident(options.table);
+  const id = ident(options.idColumn ?? "id");
+  const state = ident(options.stateColumn ?? "status");
+  const transition = async (recordId: string | number, from: string, to: string) =>
+    (
+      await options
+        .db()
+        .exec(`UPDATE ${table} SET ${state} = ? WHERE ${id} = ? AND ${state} = ?`, [
+          to,
+          recordId,
+          from,
+        ])
+    ).changes > 0;
+  return {
+    queue: (recordId: string | number) =>
+      transition(recordId, options.states.failed, options.states.queued),
+    claim: (recordId: string | number) =>
+      transition(recordId, options.states.queued, options.states.running),
+    complete: (recordId: string | number) =>
+      transition(recordId, options.states.running, options.states.success),
+    fail: (recordId: string | number) =>
+      transition(recordId, options.states.running, options.states.failed),
+    transition,
+  };
+}
+
+export function testQueue(queue: DurableQueue) {
+  return {
+    add: queue.add.bind(queue),
+    runNext: () => queue.drain(),
+    async runAll(limit = 1_000) {
+      let total = 0;
+      while (total < limit) {
+        const count = await queue.drain();
+        if (!count) break;
+        total += count;
+      }
+      return total;
+    },
+    get: queue.get.bind(queue),
+    status: queue.status.bind(queue),
+    history: queue.history.bind(queue),
+    shutdown: queue.shutdown.bind(queue),
+  };
+}
+
+export interface SagaStep<T> {
+  name: string;
+  run(value: T): void | Promise<void>;
+  compensate?(value: T, cause: unknown): void | Promise<void>;
+}
+
+export function defineSaga<T>(definition: { name: string; steps: readonly SagaStep<T>[] }) {
+  return {
+    name: definition.name,
+    async run(value: T): Promise<void> {
+      const completed: SagaStep<T>[] = [];
+      try {
+        for (const step of definition.steps) {
+          await step.run(value);
+          completed.push(step);
+        }
+      } catch (cause) {
+        for (const step of completed.reverse()) await step.compensate?.(value, cause);
+        throw cause;
+      }
+    },
+  };
+}

packages/queue/test/patterns.test.ts

@@ -0,0 +1,50 @@
+import { expect, test } from "bun:test";
+import { createDb } from "@wrnexus/db";
+import { sqlite } from "@wrnexus/db/sqlite";
+import {
+  createDurableQueue,
+  defineSaga,
+  drainOutbox,
+  memoryQueueStore,
+  testQueue,
+  withOutbox,
+} from "../src/index.ts";
+
+test("outbox commits intents and deterministic queue helpers dispatch them", async () => {
+  const db = createDb(sqlite(":memory:"));
+  const queue = createDurableQueue({ store: memoryQueueStore() });
+  const seen: number[] = [];
+  queue.process("email:send", async (job) => void seen.push((job.data as { id: number }).id));
+  await withOutbox(db, async ({ enqueue }) => enqueue({ name: "email:send" } as never, { id: 7 }));
+  expect(
+    await drainOutbox(db, (name) =>
+      name === "email:send"
+        ? { add: (data, options) => queue.add(name, data, options) }
+        : undefined,
+    ),
+  ).toBe(1);
+  expect(await testQueue(queue).runAll()).toBe(1);
+  expect(seen).toEqual([7]);
+});
+
+test("sagas compensate completed steps in reverse order", async () => {
+  const calls: string[] = [];
+  const saga = defineSaga({
+    name: "send",
+    steps: [
+      {
+        name: "reserve",
+        run: () => void calls.push("reserve"),
+        compensate: () => void calls.push("refund"),
+      },
+      {
+        name: "queue",
+        run: () => {
+          throw new Error("down");
+        },
+      },
+    ],
+  });
+  await expect(saga.run({})).rejects.toThrow("down");
+  expect(calls).toEqual(["reserve", "refund"]);
+});