feat(queue): add typed application queue lifecycle

packages/queue/src/durable.ts

@@ -17,10 +17,54 @@ export interface QueueStore {
   findByIdempotencyKey?(name: string, key: string): Promise<Job | null>;
   size?(): Promise<number>;
   claim?(id: string, worker: string, leaseUntil: number, now?: number): Promise<boolean>;
+  release?(id: string, worker: string): Promise<void>;
+  archive?(record: QueueJobRecord): Promise<void>;
+  history?(id?: string): Promise<QueueJobRecord[]>;
+}
+
+export type QueueJobState = "completed" | "failed" | "cancelled";
+export interface QueueJobRecord {
+  job: Job;
+  state: QueueJobState;
+  finishedAt: number;
+  error?: string;
+}
+
+export type QueueEventType =
+  | "job.added"
+  | "job.started"
+  | "job.completed"
+  | "job.retrying"
+  | "job.failed"
+  | "job.cancelled"
+  | "worker.started"
+  | "worker.stopped"
+  | "worker.error";
+
+export interface QueueEvent {
+  type: QueueEventType;
+  at: number;
+  workerId: string;
+  job?: Job;
+  error?: unknown;
+  durationMs?: number;
+}
+
+export interface QueueHealth {
+  running: boolean;
+  accepting: boolean;
+  draining: boolean;
+  active: number;
+  pending: number;
+  failed: number;
+  workerId: string;
+  lastPollAt?: number;
+  lastError?: unknown;
 }
 
 export function memoryQueueStore(): QueueStore {
   const jobs = new Map<string, Job>();
+  const records = new Map<string, QueueJobRecord>();
 
   return {
     async put(job) {
@@ -59,6 +103,14 @@ export function memoryQueueStore(): QueueStore {
     async size() {
       return jobs.size;
     },
+    async archive(record) {
+      records.set(record.job.id, structuredClone(record));
+    },
+    async history(id) {
+      return [...records.values()]
+        .filter((record) => !id || record.job.id === id)
+        .map((record) => structuredClone(record));
+    },
   };
 }
 
@@ -69,16 +121,27 @@ export interface DurableQueueOptions {
   concurrency?: number;
   capacity?: number;
   leaseMs?: number;
+  pollMs?: number;
   backoff?: (attempt: number) => number;
   now?: () => number;
   onDeadLetter?: (job: Job, error: unknown) => void | Promise<void>;
   context?: (job: Job, signal: AbortSignal) => ExecutionContext;
+  onEvent?: (event: QueueEvent) => void | Promise<void>;
+  /** Maintenance hook run before each poll, for application-specific recovery. */
+  beforeDrain?: (queue: DurableQueue) => void | Promise<void>;
 }
 
 export interface DurableQueue {
   add<T>(name: string, data: T, options?: AddOptions): Promise<Job<T>>;
   process<T>(name: string, handler: JobHandler<T>): void;
   drain(): Promise<number>;
+  start(): void;
+  stop(): void;
+  isRunning(): boolean;
+  health(): Promise<QueueHealth>;
+  get(id: string): Promise<Job | null>;
+  status(id: string): Promise<"queued" | QueueJobState | "missing">;
+  history(id?: string): Promise<QueueJobRecord[]>;
   cancel(id: string): Promise<boolean>;
   shutdown(options?: { force?: boolean }): Promise<void>;
   list(name?: string): Promise<Job[]>;
@@ -114,11 +177,19 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
   const concurrency = positiveInteger(options.concurrency ?? 10, "queue concurrency");
   const capacity = positiveInteger(options.capacity ?? 10_000, "queue capacity");
   const leaseMs = positiveInteger(options.leaseMs ?? 30_000, "queue leaseMs");
+  const pollMs = positiveInteger(options.pollMs ?? 250, "queue pollMs");
   let sequence = 0;
   let draining = false;
   let accepting = true;
+  let timer: ReturnType<typeof setInterval> | null = null;
+  let lastPollAt: number | undefined;
+  let lastError: unknown;
   const active = new Map<string, { controller: AbortController; promise: Promise<boolean> }>();
 
+  const emit = async (event: Omit<QueueEvent, "at" | "workerId">) => {
+    await options.onEvent?.({ ...event, at: now(), workerId });
+  };
+
   function beginJob(job: Job): Promise<boolean> {
     const controller = new AbortController();
     const promise = runJob(job, controller).finally(() => active.delete(job.id));
@@ -135,6 +206,8 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
     }
 
     job.attempts += 1;
+    const startedAt = now();
+    await emit({ type: "job.started", job: structuredClone(job) });
 
     try {
       await handler(job, {
@@ -153,8 +226,14 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
         job.runAt = now() + job.repeat;
         await store.put(job);
       } else {
+        await store.archive?.({ job: structuredClone(job), state: "completed", finishedAt: now() });
         await store.remove(job.id);
       }
+      await emit({
+        type: "job.completed",
+        job: structuredClone(job),
+        durationMs: now() - startedAt,
+      });
     } catch (error) {
       if (controller.signal.aborted) {
         await store.remove(job.id);
@@ -167,17 +246,25 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
         );
         job.runAt = now() + delay;
         await store.put(job);
+        await emit({ type: "job.retrying", job: structuredClone(job), error });
       } else {
         await store.remove(job.id);
         deadLetters.set(job.id, structuredClone(job));
+        await store.archive?.({
+          job: structuredClone(job),
+          state: "failed",
+          finishedAt: now(),
+          error: error instanceof Error ? error.message : String(error),
+        });
         await options.onDeadLetter?.(structuredClone(job), error);
+        await emit({ type: "job.failed", job: structuredClone(job), error });
       }
     }
 
     return true;
   }
 
-  return {
+  const durableQueue: DurableQueue = {
     async add<T>(name: string, data: T, add: AddOptions = {}) {
       if (!accepting) throw new Error("WRN-QUEUE-CLOSED: queue is shutting down");
       if (!name.trim()) throw new TypeError("queue job name cannot be empty");
@@ -216,6 +303,7 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
         createdAt,
       };
       await store.put(job);
+      await emit({ type: "job.added", job: structuredClone(job) });
       return structuredClone(job);
     },
 
@@ -233,7 +321,9 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
     async drain() {
       if (draining) return 0;
       draining = true;
+      lastPollAt = now();
       try {
+        await options.beforeDrain?.(durableQueue);
         const due = await store.due(now(), concurrency);
         const results = await Promise.all(due.map(beginJob));
         return results.filter(Boolean).length;
@@ -250,19 +340,80 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
       return store.list(name);
     },
 
+    get(id) {
+      return store.get(id);
+    },
+
+    async status(id) {
+      if (await store.get(id)) return "queued";
+      const record = (await store.history?.(id))?.at(-1);
+      if (record) return record.state;
+      if (deadLetters.has(id)) return "failed";
+      return "missing";
+    },
+
+    async history(id) {
+      return (await store.history?.(id)) ?? [];
+    },
+
+    start() {
+      if (!accepting) throw new Error("WRN-QUEUE-CLOSED: queue is shutting down");
+      if (timer) return;
+      timer = setInterval(() => {
+        void this.drain().catch(async (error: unknown) => {
+          lastError = error;
+          await emit({ type: "worker.error", error });
+        });
+      }, pollMs);
+      timer.unref?.();
+      void emit({ type: "worker.started" });
+    },
+
+    stop() {
+      if (timer) clearInterval(timer);
+      timer = null;
+      void emit({ type: "worker.stopped" });
+    },
+
+    isRunning() {
+      return timer !== null;
+    },
+
+    async health() {
+      return {
+        running: timer !== null,
+        accepting,
+        draining,
+        active: active.size,
+        pending: store.size ? await store.size() : (await store.list()).length,
+        failed: deadLetters.size,
+        workerId,
+        lastPollAt,
+        lastError,
+      };
+    },
+
     async cancel(id) {
       const running = active.get(id);
       if (running) {
         running.controller.abort();
+        const job = await store.get(id);
+        if (job) await store.archive?.({ job, state: "cancelled", finishedAt: now() });
+        await emit({ type: "job.cancelled" });
         return true;
       }
       if (!(await store.get(id))) return false;
+      const job = await store.get(id);
+      if (job) await store.archive?.({ job, state: "cancelled", finishedAt: now() });
       await store.remove(id);
+      await emit({ type: "job.cancelled" });
       return true;
     },
 
     async shutdown(shutdownOptions = {}) {
       accepting = false;
+      if (timer) clearInterval(timer);
+      timer = null;
       if (shutdownOptions.force) {
         for (const { controller } of active.values()) controller.abort();
       }
@@ -270,7 +421,13 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
     },
 
     async retry(id) {
-      const job = deadLetters.get(id);
+      let job = deadLetters.get(id);
+      if (!job) {
+        const record = (await store.history?.(id))?.find(
+          (candidate) => candidate.state === "failed",
+        );
+        job = record?.job;
+      }
       if (!job) return false;
       deadLetters.delete(id);
       job.attempts = 0;
@@ -279,4 +436,5 @@ export function createDurableQueue(options: DurableQueueOptions = {}): DurableQu
       return true;
     },
   };
+  return durableQueue;
 }