feat: make queues durable by default and add seed helpers
Quality / quality (ubuntu-latest) (push) Failing after 11m1s
Quality / quality (windows-latest) (push) Canceled after 0s

This commit is contained in:
2026-08-23 09:50:25 +05:30
parent 8fb96f521f
commit 46195462c3
22 changed files with 283 additions and 15 deletions
+89
View File
@@ -0,0 +1,89 @@
import { getDb, hasDb, registerDb, type Db } from "@wrnexus/db";
import { connectFromConfig } from "@wrnexus/db/connect";
import { memoryQueueStore, type QueueStore } from "./durable.ts";
import { installSqliteQueueSchema, sqliteQueueStore } from "./sqlite.ts";
export type QueueStorage = "sqlite" | "database" | "memory";
export interface QueueStorageConfig {
/** Durable SQLite is the default. Use memory only for disposable/test queues. */
storage?: QueueStorage;
/** Named `databases` connection, or `default` for the app's `db`. */
databaseName?: string;
table?: string;
/** Used internally to resolve the default `.wrnexus/queue.sqlite` path. */
appRoot?: string;
}
let configured: QueueStorageConfig = { storage: "sqlite" };
export function configureQueueStorage(config: QueueStorageConfig = {}): void {
configured = { storage: "sqlite", ...config };
}
export function queueStorageConfig(): Readonly<QueueStorageConfig> {
return { ...configured };
}
function lazyStore(resolve: () => Promise<QueueStore>): QueueStore {
let pending: Promise<QueueStore> | undefined;
const ready = () => (pending ??= resolve());
return {
async put(job) { return (await ready()).put(job); },
async get(id) { return (await ready()).get(id); },
async remove(id) { return (await ready()).remove(id); },
async due(now, limit) { return (await ready()).due(now, limit); },
async list(name) { return (await ready()).list(name); },
async findByIdempotencyKey(name, key) {
return (await ready()).findByIdempotencyKey?.(name, key) ?? null;
},
async size() { return (await ready()).size?.() ?? (await (await ready()).list()).length; },
async claim(id, worker, leaseUntil, now) {
return (await ready()).claim?.(id, worker, leaseUntil, now) ?? true;
},
async release(id, worker) { await (await ready()).release?.(id, worker); },
async archive(record) { await (await ready()).archive?.(record); },
async history(id) { return (await ready()).history?.(id) ?? []; },
};
}
async function resolveConfiguredStore(): Promise<QueueStore> {
const config = configured;
if (config.storage === "memory") return memoryQueueStore();
const databaseName = config.databaseName ?? "default";
let db: Db;
if (config.storage === "database") {
if (!hasDb(databaseName)) {
throw new Error(
`WRN-QUEUE-DATABASE: database '${databaseName}' is not configured. ` +
"Add it to db/databases or choose storage: 'sqlite'.",
);
}
db = getDb(databaseName);
} else {
if (hasDb("__wrnexus_queue")) db = getDb("__wrnexus_queue");
else {
db = connectFromConfig(
{ driver: "sqlite", url: "file:./.wrnexus/queue.sqlite" },
config.appRoot ?? process.cwd(),
);
registerDb("__wrnexus_queue", db);
}
}
if (db.driver.dialect !== "sqlite") {
throw new Error(
`WRN-QUEUE-DATABASE: configured queue storage currently requires SQLite; ` +
`database '${databaseName}' uses ${db.driver.dialect}. Pass a custom queue store for that driver.`,
);
}
const table = config.table ?? "wrnexus_jobs";
await installSqliteQueueSchema(db, table);
return sqliteQueueStore(db, table);
}
/** Lazy global store used by defineQueue; runtime config is read on first operation. */
export function configuredQueueStore(): QueueStore {
return lazyStore(resolveConfiguredStore);
}
+4
View File
@@ -5,11 +5,13 @@ import {
type DurableQueueOptions,
type QueueHealth,
} from "./durable.ts";
import { configuredQueueStore } from "./configured.ts";
export type JobStatus = "queued" | "completed" | "failed" | "cancelled" | "missing";
export interface DefinedJobOptions<T> extends Omit<AddOptions, "idempotencyKey"> {
run: (data: T, context: JobContext & { job: Job<T> }) => void | Promise<void>;
success?: (data: T, job: Job<T>) => void | Promise<void>;
failed?: (data: T, error: unknown, job: Job<T>) => void | Promise<void>;
idempotency?: (data: T) => string | undefined;
validate?: (data: unknown) => data is T;
@@ -65,6 +67,7 @@ export function defineQueue<TJobs extends QueueJobDefinitions>(
definition.queue ??
createDurableQueue({
...definition.options,
store: definition.options?.store ?? configuredQueueStore(),
async onDeadLetter(job, error) {
await definition.options?.onDeadLetter?.(job, error);
const prefix = `${name}:`;
@@ -82,6 +85,7 @@ export function defineQueue<TJobs extends QueueJobDefinitions>(
throw new TypeError(`WRN-QUEUE-PAYLOAD: invalid payload for '${jobName}'`);
}
await jobDefinition.run(job.data, { ...context, job });
await jobDefinition.success?.(job.data, job);
};
queue.process(jobName, handler);
+2
View File
@@ -317,6 +317,8 @@ export type {
} from "./defined.ts";
export { installSqliteQueueSchema, sqliteQueueSchema, sqliteQueueStore } from "./sqlite.ts";
export type { SqliteQueueClient } from "./sqlite.ts";
export { configureQueueStorage, configuredQueueStore, queueStorageConfig } from "./configured.ts";
export type { QueueStorage, QueueStorageConfig } from "./configured.ts";
export { redisQueueStore, postgresQueueStore, POSTGRES_QUEUE_SCHEMA } from "./stores.ts";
export type { RedisQueueClient, SqlQueueClient } from "./stores.ts";
export {