feat(queue): add typed application queue lifecycle

packages/queue/src/defined.ts

@@ -0,0 +1,139 @@
+import type { AddOptions, Job, JobContext, JobHandler } from "./index.ts";
+import {
+  createDurableQueue,
+  type DurableQueue,
+  type DurableQueueOptions,
+  type QueueHealth,
+} from "./durable.ts";
+
+export type JobStatus = "queued" | "completed" | "failed" | "cancelled" | "missing";
+
+export interface DefinedJobOptions<T> extends Omit<AddOptions, "idempotencyKey"> {
+  run: (data: T, context: JobContext & { job: Job<T> }) => void | Promise<void>;
+  failed?: (data: T, error: unknown, job: Job<T>) => void | Promise<void>;
+  idempotency?: (data: T) => string | undefined;
+  validate?: (data: unknown) => data is T;
+}
+
+export type QueueJobDefinitions = Record<string, DefinedJobOptions<any>>;
+
+export interface DefineQueueOptions<TJobs extends QueueJobDefinitions> {
+  name: string;
+  jobs: TJobs;
+  queue?: DurableQueue;
+  options?: DurableQueueOptions;
+}
+
+type DataOf<T> = T extends DefinedJobOptions<infer I> ? I : never;
+
+export interface DefinedJob<T> {
+  readonly name: string;
+  add(data: T, options?: AddOptions): Promise<Job<T>>;
+  addMany(entries: readonly { data: T; options?: AddOptions }[]): Promise<Job<T>[]>;
+  get(id: string): Promise<Job<T> | null>;
+  status(id: string): Promise<JobStatus>;
+  list(): Promise<Job<T>[]>;
+  cancel(id: string): Promise<boolean>;
+  retry(id: string): Promise<boolean>;
+}
+
+export type DefinedJobs<TJobs extends QueueJobDefinitions> = {
+  [K in keyof TJobs]: DefinedJob<DataOf<TJobs[K]>>;
+};
+
+export interface DefinedQueue<TJobs extends QueueJobDefinitions> {
+  readonly name: string;
+  readonly jobs: DefinedJobs<TJobs>;
+  readonly raw: DurableQueue;
+  start(): void;
+  stop(): void;
+  shutdown(options?: { force?: boolean }): Promise<void>;
+  health(): Promise<QueueHealth>;
+  drain(): Promise<number>;
+}
+
+function fullName(queue: string, job: string): string {
+  return `${queue}:${job}`;
+}
+
+export function defineQueue<TJobs extends QueueJobDefinitions>(
+  definition: DefineQueueOptions<TJobs>,
+): DefinedQueue<TJobs> & DefinedJobs<TJobs> {
+  const name = definition.name.trim();
+  if (!name) throw new TypeError("queue name cannot be empty");
+  const queue =
+    definition.queue ??
+    createDurableQueue({
+      ...definition.options,
+      async onDeadLetter(job, error) {
+        await definition.options?.onDeadLetter?.(job, error);
+        const prefix = `${name}:`;
+        if (!job.name.startsWith(prefix)) return;
+        const failed = definition.jobs[job.name.slice(prefix.length)]?.failed;
+        await failed?.(job.data, error, job);
+      },
+    });
+  const jobs: Record<string, DefinedJob<unknown>> = {};
+
+  for (const [shortName, jobDefinition] of Object.entries(definition.jobs)) {
+    const jobName = fullName(name, shortName);
+    const handler: JobHandler<unknown> = async (job, context) => {
+      if (jobDefinition.validate && !jobDefinition.validate(job.data)) {
+        throw new TypeError(`WRN-QUEUE-PAYLOAD: invalid payload for '${jobName}'`);
+      }
+      await jobDefinition.run(job.data, { ...context, job });
+    };
+    queue.process(jobName, handler);
+
+    jobs[shortName] = {
+      name: jobName,
+      add(data, options = {}) {
+        if (jobDefinition.validate && !jobDefinition.validate(data)) {
+          return Promise.reject(
+            new TypeError(`WRN-QUEUE-PAYLOAD: invalid payload for '${jobName}'`),
+          );
+        }
+        return queue.add(jobName, data, {
+          maxAttempts: jobDefinition.maxAttempts,
+          delayMs: jobDefinition.delayMs,
+          repeat: jobDefinition.repeat,
+          priority: jobDefinition.priority,
+          idempotencyKey: jobDefinition.idempotency?.(data),
+          ...options,
+        });
+      },
+      addMany(entries) {
+        return Promise.all(entries.map((entry) => this.add(entry.data, entry.options)));
+      },
+      async get(id) {
+        const job = await queue.get(id);
+        return job?.name === jobName ? (job as Job<unknown>) : null;
+      },
+      async status(id) {
+        return queue.status(id);
+      },
+      async list() {
+        return (await queue.list(jobName)) as Job<unknown>[];
+      },
+      cancel(id) {
+        return queue.cancel(id);
+      },
+      retry(id) {
+        return queue.retry(id);
+      },
+    };
+  }
+
+  const result = {
+    name,
+    jobs: jobs as DefinedJobs<TJobs>,
+    raw: queue,
+    start: () => queue.start(),
+    stop: () => queue.stop(),
+    shutdown: (options?: { force?: boolean }) => queue.shutdown(options),
+    health: () => queue.health(),
+    drain: () => queue.drain(),
+    ...(jobs as DefinedJobs<TJobs>),
+  };
+  return result;
+}

packages/queue/src/durable.ts

@@ -17,10 +17,54 @@ export interface QueueStore {
   findByIdempotencyKey?(name: string, key: string): Promise<Job | null>;
   size?(): Promise<number>;
   claim?(id: string, worker: string, leaseUntil: number, now?: number): Promise<boolean>;
+  release?(id: string, worker: string): Promise<void>;
+  archive?(record: QueueJobRecord): Promise<void>;
+  history?(id?: string): Promise<QueueJobRecord[]>;
+}
+
+export type QueueJobState = "completed" | "failed" | "cancelled";
+export interface QueueJobRecord {
+  job: Job;
+  state: QueueJobState;
+  finishedAt: number;
+  error?: string;
+}
+
+export type QueueEventType =
+  | "job.added"
+  | "job.started"
+  | "job.completed"
+  | "job.retrying"
+  | "job.failed"
+  | "job.cancelled"
+  | "worker.started"
+  | "worker.stopped"
+  | "worker.error";
+
+export interface QueueEvent {
+  type: QueueEventType;
+  at: number;
+  workerId: string;
+  job?: Job;
+  error?: unknown;
+  durationMs?: number;
+}
+
+export interface QueueHealth {
+  running: boolean;
+  accepting: boolean;
+  draining: boolean;
+  active: number;
+  pending: number;
+  failed: number;
+  workerId: string;
+  lastPollAt?: number;
+  lastError?: unknown;
 }
 
 export function memoryQueueStore(): QueueStore {
   const jobs = new Map<string, Job>();
+  const records = new Map<string, QueueJobRecord>();
 
   return {
     async put(job) {
@@ -59,6 +103,14 @@ export function memoryQueueStore(): QueueStore {
     async size() {
       return jobs.size;
     },
+    async archive(record) {
+      records.set(record.job.id, structuredClone(record));
+    },
+    async history(id) {
+      return [...records.values()]
+        .filter((record) => !id || record.job.id === id)
+        .map((record) => structuredClone(record));
+    },
   };
 }
 
@@ -69,16 +121,27 @@ export interface DurableQueueOptions {
   concurrency?: number;
   capacity?: number;
   leaseMs?: number;
+  pollMs?: number;
   backoff?: (attempt: number) => number;
   now?: () => number;
   onDeadLetter?: (job: Job, error: unknown) => void | Promise<void>;
   context?: (job: Job, signal: AbortSignal) => ExecutionContext;
+  onEvent?: (event: QueueEvent) => void | Promise<void>;
+  /** Maintenance hook run before each poll, for application-specific recovery. */
+  beforeDrain?: (queue: DurableQueue) => void | Promise<void>;
 }
 
 export interface DurableQueue {
   add<T>(name: string, data: T, options?: AddOptions): Promise<Job<T>>;
   process<T>(name: string, handler: JobHandler<T>): void;
   drain(): Promise<number>;
+  start(): void;
+  stop(): void;
+  isRunning(): boolean;
+  health(): Promise<QueueHealth>;
+  get(id: string): Promise<Job | null>;
+  status(id: string): Promise<"queued" | QueueJobState | "missing">;
+  history(id?: string): Promise<QueueJobRecord[]>;
   cancel(id: string): Promise<boolean>;
   shutdown(options?: { force?: boolean }): Promise<void>;
   list(name?: string): Promise<Job[]>;
@@ -114,11 +177,19 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
   const concurrency = positiveInteger(options.concurrency ?? 10, "queue concurrency");
   const capacity = positiveInteger(options.capacity ?? 10_000, "queue capacity");
   const leaseMs = positiveInteger(options.leaseMs ?? 30_000, "queue leaseMs");
+  const pollMs = positiveInteger(options.pollMs ?? 250, "queue pollMs");
   let sequence = 0;
   let draining = false;
   let accepting = true;
+  let timer: ReturnType<typeof setInterval> | null = null;
+  let lastPollAt: number | undefined;
+  let lastError: unknown;
   const active = new Map<string, { controller: AbortController; promise: Promise<boolean> }>();
 
+  const emit = async (event: Omit<QueueEvent, "at" | "workerId">) => {
+    await options.onEvent?.({ ...event, at: now(), workerId });
+  };
+
   function beginJob(job: Job): Promise<boolean> {
     const controller = new AbortController();
     const promise = runJob(job, controller).finally(() => active.delete(job.id));
@@ -135,6 +206,8 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
     }
 
     job.attempts += 1;
+    const startedAt = now();
+    void emit({ type: "job.started", job: structuredClone(job) });
 
     try {
       await handler(job, {
@@ -153,8 +226,14 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
         job.runAt = now() + job.repeat;
         await store.put(job);
       } else {
+        await store.archive?.({ job: structuredClone(job), state: "completed", finishedAt: now() });
         await store.remove(job.id);
       }
+      await emit({
+        type: "job.completed",
+        job: structuredClone(job),
+        durationMs: now() - startedAt,
+      });
     } catch (error) {
       if (controller.signal.aborted) {
         await store.remove(job.id);
@@ -167,17 +246,25 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
         );
         job.runAt = now() + delay;
         await store.put(job);
+        await emit({ type: "job.retrying", job: structuredClone(job), error });
       } else {
         await store.remove(job.id);
         deadLetters.set(job.id, structuredClone(job));
+        await store.archive?.({
+          job: structuredClone(job),
+          state: "failed",
+          finishedAt: now(),
+          error: error instanceof Error ? error.message : String(error),
+        });
         await options.onDeadLetter?.(structuredClone(job), error);
+        await emit({ type: "job.failed", job: structuredClone(job), error });
       }
     }
 
     return true;
   }
 
-  return {
+  const durableQueue: DurableQueue = {
     async add<T>(name: string, data: T, add: AddOptions = {}) {
       if (!accepting) throw new Error("WRN-QUEUE-CLOSED: queue is shutting down");
       if (!name.trim()) throw new TypeError("queue job name cannot be empty");
@@ -216,6 +303,7 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
         createdAt,
       };
       await store.put(job);
+      await emit({ type: "job.added", job: structuredClone(job) });
       return structuredClone(job);
     },
 
@@ -233,7 +321,9 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
     async drain() {
       if (draining) return 0;
       draining = true;
+      lastPollAt = now();
       try {
+        if (options.beforeDrain) await options.beforeDrain(durableQueue);
         const due = await store.due(now(), concurrency);
         const results = await Promise.all(due.map(beginJob));
         return results.filter(Boolean).length;
@@ -250,19 +340,80 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
       return store.list(name);
     },
 
+    get(id) {
+      return store.get(id);
+    },
+
+    async status(id) {
+      if (await store.get(id)) return "queued";
+      const record = (await store.history?.(id))?.at(-1);
+      if (record) return record.state;
+      if (deadLetters.has(id)) return "failed";
+      return "missing";
+    },
+
+    async history(id) {
+      return (await store.history?.(id)) ?? [];
+    },
+
+    start() {
+      if (!accepting) throw new Error("WRN-QUEUE-CLOSED: queue is shutting down");
+      if (timer) return;
+      timer = setInterval(() => {
+        void this.drain().catch(async (error: unknown) => {
+          lastError = error;
+          await emit({ type: "worker.error", error });
+        });
+      }, pollMs);
+      timer.unref?.();
+      void emit({ type: "worker.started" });
+    },
+
+    stop() {
+      if (timer) clearInterval(timer);
+      timer = null;
+      void emit({ type: "worker.stopped" });
+    },
+
+    isRunning() {
+      return timer !== null;
+    },
+
+    async health() {
+      return {
+        running: timer !== null,
+        accepting,
+        draining,
+        active: active.size,
+        pending: store.size ? await store.size() : (await store.list()).length,
+        failed: deadLetters.size,
+        workerId,
+        lastPollAt,
+        lastError,
+      };
+    },
+
     async cancel(id) {
       const running = active.get(id);
       if (running) {
         running.controller.abort();
+        const job = await store.get(id);
+        if (job) await store.archive?.({ job, state: "cancelled", finishedAt: now() });
+        await emit({ type: "job.cancelled" });
         return true;
       }
       if (!(await store.get(id))) return false;
+      const job = await store.get(id);
+      if (job) await store.archive?.({ job, state: "cancelled", finishedAt: now() });
       await store.remove(id);
+      await emit({ type: "job.cancelled" });
       return true;
     },
 
     async shutdown(shutdownOptions = {}) {
       accepting = false;
+      if (timer) clearInterval(timer);
+      timer = null;
       if (shutdownOptions.force) {
         for (const { controller } of active.values()) controller.abort();
       }
@@ -270,7 +421,13 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
     },
 
     async retry(id) {
-      const job = deadLetters.get(id);
+      let job = deadLetters.get(id);
+      if (!job) {
+        const record = (await store.history?.(id))?.find(
+          (candidate) => candidate.state === "failed",
+        );
+        job = record?.job;
+      }
       if (!job) return false;
       deadLetters.delete(id);
       job.attempts = 0;
@@ -279,4 +436,5 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
       return true;
     },
   };
+  return durableQueue;
 }

packages/queue/src/index.ts

@@ -298,6 +298,25 @@ export function createQueue(options: QueueOptions = {}): Queue {
 }
 export { memoryQueueStore, createDurableQueue } from "./durable.ts";
 export type { QueueStore, DurableQueue, DurableQueueOptions } from "./durable.ts";
+export type {
+  QueueEvent,
+  QueueEventType,
+  QueueHealth,
+  QueueJobRecord,
+  QueueJobState,
+} from "./durable.ts";
+export { defineQueue } from "./defined.ts";
+export type {
+  DefinedJob,
+  DefinedJobOptions,
+  DefinedJobs,
+  DefinedQueue,
+  DefineQueueOptions,
+  JobStatus,
+  QueueJobDefinitions,
+} from "./defined.ts";
+export { installSqliteQueueSchema, sqliteQueueSchema, sqliteQueueStore } from "./sqlite.ts";
+export type { SqliteQueueClient } from "./sqlite.ts";
 export { redisQueueStore, postgresQueueStore, POSTGRES_QUEUE_SCHEMA } from "./stores.ts";
 export type { RedisQueueClient, SqlQueueClient } from "./stores.ts";
 export {

packages/queue/src/scheduler.ts

@@ -85,10 +85,11 @@ export interface QueueDashboardSnapshot {
 
 export async function queueDashboardSnapshot(queue: DurableQueue): Promise<QueueDashboardSnapshot> {
   const jobs = await queue.list();
+  const history = await queue.history();
   return {
     generatedAt: Date.now(),
     pending: jobs.length,
-    failed: queue.failed().length,
+    failed: history.filter((record) => record.state === "failed").length || queue.failed().length,
     byName: jobs.reduce<Record<string, number>>((counts, job) => {
       counts[job.name] = (counts[job.name] ?? 0) + 1;
       return counts;

packages/queue/src/sqlite.ts

@@ -0,0 +1,167 @@
+import type { Job } from "./index.ts";
+import type { QueueJobRecord, QueueStore } from "./durable.ts";
+
+export interface SqliteQueueClient {
+  exec(sql: string, parameters?: unknown[]): Promise<{ changes: number }>;
+  one<T>(sql: string, parameters?: unknown[]): Promise<T | null>;
+  all<T>(sql: string, parameters?: unknown[]): Promise<T[]>;
+}
+
+interface QueueJobRow {
+  id: string;
+  name: string;
+  payload: string;
+  run_at: number;
+  priority: number;
+  idempotency_key: string | null;
+}
+
+const SAFE_TABLE = /^[a-z_][a-z0-9_]*$/i;
+
+export function sqliteQueueSchema(table = "wrnexus_jobs"): string {
+  if (!SAFE_TABLE.test(table)) throw new Error("Invalid queue table name");
+  return `CREATE TABLE IF NOT EXISTS ${table} (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    run_at INTEGER NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 0,
+    idempotency_key TEXT,
+    lease_owner TEXT,
+    lease_until INTEGER
+  );
+  CREATE INDEX IF NOT EXISTS ${table}_due ON ${table} (run_at, priority DESC);
+  CREATE UNIQUE INDEX IF NOT EXISTS ${table}_idempotency ON ${table} (name, idempotency_key)
+    WHERE idempotency_key IS NOT NULL;
+  CREATE TABLE IF NOT EXISTS ${table}_history (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    state TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    error TEXT,
+    finished_at INTEGER NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS ${table}_history_finished ON ${table}_history (finished_at DESC);`;
+}
+
+export async function installSqliteQueueSchema(
+  db: SqliteQueueClient,
+  table = "wrnexus_jobs",
+): Promise<void> {
+  for (const statement of sqliteQueueSchema(table)
+    .split(";")
+    .map((value) => value.trim())) {
+    if (statement) await db.exec(statement);
+  }
+}
+
+export function sqliteQueueStore(db: SqliteQueueClient, table = "wrnexus_jobs"): QueueStore {
+  if (!SAFE_TABLE.test(table)) throw new Error("Invalid queue table name");
+  const decode = (row: QueueJobRow): Job => JSON.parse(row.payload) as Job;
+  return {
+    async put(job) {
+      await db.exec(
+        `INSERT INTO ${table} (id,name,payload,run_at,priority,idempotency_key,lease_owner,lease_until)
+         VALUES (?,?,?,?,?,?,NULL,NULL)
+         ON CONFLICT(id) DO UPDATE SET name=excluded.name,payload=excluded.payload,
+         run_at=excluded.run_at,priority=excluded.priority,idempotency_key=excluded.idempotency_key,
+         lease_owner=NULL,lease_until=NULL`,
+        [
+          job.id,
+          job.name,
+          JSON.stringify(job),
+          job.runAt,
+          job.priority,
+          job.idempotencyKey ?? null,
+        ],
+      );
+    },
+    async get(id) {
+      const row = await db.one<QueueJobRow>(`SELECT * FROM ${table} WHERE id = ?`, [id]);
+      return row ? decode(row) : null;
+    },
+    async remove(id) {
+      await db.exec(`DELETE FROM ${table} WHERE id = ?`, [id]);
+    },
+    async due(now, limit) {
+      const rows = await db.all<QueueJobRow>(
+        `SELECT * FROM ${table} WHERE run_at <= ? AND (lease_until IS NULL OR lease_until < ?)
+         ORDER BY priority DESC,run_at ASC LIMIT ?`,
+        [now, now, limit],
+      );
+      return rows.map(decode);
+    },
+    async list(name) {
+      const rows = name
+        ? await db.all<QueueJobRow>(`SELECT * FROM ${table} WHERE name = ? ORDER BY run_at`, [name])
+        : await db.all<QueueJobRow>(`SELECT * FROM ${table} ORDER BY run_at`);
+      return rows.map(decode);
+    },
+    async findByIdempotencyKey(name, key) {
+      const row = await db.one<QueueJobRow>(
+        `SELECT * FROM ${table} WHERE name = ? AND idempotency_key = ? LIMIT 1`,
+        [name, key],
+      );
+      return row ? decode(row) : null;
+    },
+    async size() {
+      const row = await db.one<{ total: number }>(`SELECT COUNT(*) AS total FROM ${table}`);
+      return Number(row?.total ?? 0);
+    },
+    async claim(id, worker, leaseUntil, now = Date.now()) {
+      const result = await db.exec(
+        `UPDATE ${table} SET lease_owner = ?,lease_until = ?
+         WHERE id = ? AND (lease_until IS NULL OR lease_until < ?)`,
+        [worker, leaseUntil, id, now],
+      );
+      return result.changes > 0;
+    },
+    async release(id, worker) {
+      await db.exec(
+        `UPDATE ${table} SET lease_owner = NULL,lease_until = NULL WHERE id = ? AND lease_owner = ?`,
+        [id, worker],
+      );
+    },
+    async archive(record) {
+      await db.exec(
+        `INSERT INTO ${table}_history (id,name,state,payload,error,finished_at) VALUES (?,?,?,?,?,?)
+         ON CONFLICT(id) DO UPDATE SET state=excluded.state,payload=excluded.payload,
+         error=excluded.error,finished_at=excluded.finished_at`,
+        [
+          record.job.id,
+          record.job.name,
+          record.state,
+          JSON.stringify(record.job),
+          record.error ?? null,
+          record.finishedAt,
+        ],
+      );
+    },
+    async history(id) {
+      const rows = id
+        ? await db.all<{
+            state: QueueJobRecord["state"];
+            payload: string;
+            error: string | null;
+            finished_at: number;
+          }>(
+            `SELECT state,payload,error,finished_at FROM ${table}_history WHERE id = ? ORDER BY finished_at DESC`,
+            [id],
+          )
+        : await db.all<{
+            state: QueueJobRecord["state"];
+            payload: string;
+            error: string | null;
+            finished_at: number;
+          }>(
+            `SELECT state,payload,error,finished_at FROM ${table}_history ORDER BY finished_at DESC`,
+          );
+      return rows.map((row) => ({
+        job: JSON.parse(row.payload) as Job,
+        state: row.state,
+        finishedAt: row.finished_at,
+        error: row.error ?? undefined,
+      }));
+    },
+  };
+}