release: WRNexusJS 0.8.0

packages/uploader/src/helpers.ts

@@ -0,0 +1,58 @@
+import type { UploadedFile } from "./upload.ts";
+
+export function formatFileSize(bytes: number, locale = "en"): string {
+  const value = Math.max(0, Number(bytes) || 0);
+  if (value < 1_024) return `${value} B`;
+  const units = ["KB", "MB", "GB", "TB"];
+  let current = value / 1_024;
+  let unit = units[0]!;
+  for (let index = 1; index < units.length && current >= 1_024; index++) {
+    current /= 1_024;
+    unit = units[index]!;
+  }
+  return `${new Intl.NumberFormat(locale, { maximumFractionDigits: current < 10 ? 1 : 0 }).format(current)} ${unit}`;
+}
+
+export function uploadAccept(value: string | readonly string[]): string {
+  return (typeof value === "string" ? value.split(",") : [...value])
+    .map((entry) => entry.trim())
+    .filter(Boolean)
+    .join(",");
+}
+
+export function uploadedFileMap(files: readonly UploadedFile[]): Record<string, UploadedFile> {
+  return Object.fromEntries(files.map((file) => [file.key, file]));
+}
+
+export function uploaderAttributes(
+  options: {
+    store?: string;
+    endpoint?: string;
+    accept?: string | readonly string[];
+    maxBytes?: number;
+    multiple?: boolean;
+    field?: string;
+    label?: string;
+  } = {},
+): Record<string, string | boolean> {
+  return {
+    "data-uploader": options.store ?? "default",
+    "data-endpoint": options.endpoint ?? "/api/upload",
+    ...(options.accept ? { "data-accept": uploadAccept(options.accept) } : {}),
+    ...(options.maxBytes ? { "data-max": String(options.maxBytes) } : {}),
+    ...(options.multiple ? { "data-multiple": true } : {}),
+    ...(options.field ? { "data-field": options.field } : {}),
+    ...(options.label ? { "data-label": options.label } : {}),
+  };
+}
+
+export function assertUploadedFiles(
+  files: readonly UploadedFile[],
+  options: { min?: number; max?: number } = {},
+): readonly UploadedFile[] {
+  const min = Math.max(0, options.min ?? 0);
+  const max = Math.max(min, options.max ?? Number.POSITIVE_INFINITY);
+  if (files.length < min) throw new Error(`WRN-UPLOAD-MIN-FILES: expected at least ${min}`);
+  if (files.length > max) throw new Error(`WRN-UPLOAD-MAX-FILES: expected at most ${max}`);
+  return files;
+}

packages/uploader/src/index.ts

@@ -38,7 +38,7 @@ export {
   UploadError,
   UPLOADS_PREFIX,
 } from "./upload.ts";
-export type { UploadedFile, UploadOptions } from "./upload.ts";
+export type { UploadedFile, UploadOptions, UploadScanInput, UploadScanResult } from "./upload.ts";
 
 export { configureStorage, getStore, hasStorage, storeNames } from "./client.ts";
 export type { Store } from "./client.ts";
@@ -70,3 +70,37 @@ export {
   UploadPolicyError,
 } from "./security.ts";
 export type { UploadPolicy, UploadInspection, SignedFileToken } from "./security.ts";
+export {
+  formatFileSize,
+  uploadAccept,
+  uploadedFileMap,
+  uploaderAttributes,
+  assertUploadedFiles,
+} from "./helpers.ts";
+export { uploaderPlugin, uploaderComponentsDir } from "./plugin.ts";
+export type { UploaderPluginOptions } from "./plugin.ts";
+export { createResumableUploadManager, memoryResumableSessionStore } from "./resumable.ts";
+export type {
+  ResumableUploadManager,
+  ResumableUploadManagerOptions,
+  ResumableUploadSession,
+  ResumableSessionStore,
+  CreateResumableUpload,
+  ResumableChunkResult,
+} from "./resumable.ts";
+export {
+  memoryQuotaStore,
+  postgresQuotaStore,
+  POSTGRES_QUOTA_SCHEMA,
+  multipartUpload,
+  createTemporaryObjectCleaner,
+  ffmpegVideoTranscoder,
+} from "./operations.ts";
+export type {
+  QuotaUsage,
+  QuotaStore,
+  QuotaSqlClient,
+  MultipartObjectClient,
+  TemporaryObject,
+  VideoTranscodeOptions,
+} from "./operations.ts";

packages/uploader/src/operations.ts

@@ -0,0 +1,200 @@
+import type { PutMeta, StorageDriver } from "./driver.ts";
+
+export interface QuotaUsage {
+  owner: string;
+  bytes: number;
+  objects: number;
+  updatedAt: number;
+}
+export interface QuotaStore {
+  get(owner: string): Promise<QuotaUsage>;
+  reserve(
+    owner: string,
+    bytes: number,
+    limits: { bytes: number; objects?: number },
+  ): Promise<boolean>;
+  release(owner: string, bytes: number): Promise<void>;
+}
+
+export function memoryQuotaStore(): QuotaStore {
+  const values = new Map<string, QuotaUsage>();
+  return {
+    async get(owner) {
+      return structuredClone(
+        values.get(owner) ?? { owner, bytes: 0, objects: 0, updatedAt: Date.now() },
+      );
+    },
+    async reserve(owner, bytes, limits) {
+      if (!Number.isInteger(bytes) || bytes < 0)
+        throw new RangeError("Quota bytes must be non-negative");
+      const current = values.get(owner) ?? { owner, bytes: 0, objects: 0, updatedAt: Date.now() };
+      if (
+        current.bytes + bytes > limits.bytes ||
+        current.objects + 1 > (limits.objects ?? Number.MAX_SAFE_INTEGER)
+      )
+        return false;
+      values.set(owner, {
+        owner,
+        bytes: current.bytes + bytes,
+        objects: current.objects + 1,
+        updatedAt: Date.now(),
+      });
+      return true;
+    },
+    async release(owner, bytes) {
+      const current = values.get(owner);
+      if (!current) return;
+      values.set(owner, {
+        ...current,
+        bytes: Math.max(0, current.bytes - Math.max(0, bytes)),
+        objects: Math.max(0, current.objects - 1),
+        updatedAt: Date.now(),
+      });
+    },
+  };
+}
+
+export interface QuotaSqlClient {
+  query<T = any>(sql: string, parameters?: unknown[]): Promise<{ rows: T[] }>;
+}
+
+/** PostgreSQL quota accounting using a single atomic conditional upsert. */
+export function postgresQuotaStore(
+  db: QuotaSqlClient,
+  table = "wrnexus_storage_quota",
+): QuotaStore {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) throw new Error("Invalid quota table name");
+  return {
+    async get(owner) {
+      const result = await db.query<QuotaUsage>(
+        `SELECT owner,bytes,objects,updated_at AS "updatedAt" FROM ${table} WHERE owner=$1`,
+        [owner],
+      );
+      return result.rows[0] ?? { owner, bytes: 0, objects: 0, updatedAt: Date.now() };
+    },
+    async reserve(owner, bytes, limits) {
+      const result = await db.query(
+        `INSERT INTO ${table} (owner,bytes,objects,updated_at) VALUES ($1,$2,1,$5) ON CONFLICT (owner) DO UPDATE SET bytes=${table}.bytes+$2,objects=${table}.objects+1,updated_at=$5 WHERE ${table}.bytes+$2 <= $3 AND ${table}.objects+1 <= $4 RETURNING owner`,
+        [owner, bytes, limits.bytes, limits.objects ?? 2_147_483_647, Date.now()],
+      );
+      return result.rows.length === 1;
+    },
+    async release(owner, bytes) {
+      await db.query(
+        `UPDATE ${table} SET bytes=GREATEST(0,bytes-$2),objects=GREATEST(0,objects-1),updated_at=$3 WHERE owner=$1`,
+        [owner, bytes, Date.now()],
+      );
+    },
+  };
+}
+
+export const POSTGRES_QUOTA_SCHEMA = `CREATE TABLE IF NOT EXISTS wrnexus_storage_quota (owner text PRIMARY KEY, bytes bigint NOT NULL DEFAULT 0, objects integer NOT NULL DEFAULT 0, updated_at bigint NOT NULL);`;
+
+export interface MultipartObjectClient {
+  create(key: string, meta: PutMeta): Promise<string>;
+  uploadPart(uploadId: string, key: string, part: number, bytes: Uint8Array): Promise<string>;
+  complete(
+    uploadId: string,
+    key: string,
+    parts: Array<{ part: number; etag: string }>,
+  ): Promise<void>;
+  abort(uploadId: string, key: string): Promise<void>;
+}
+
+export async function multipartUpload(
+  client: MultipartObjectClient,
+  key: string,
+  bytes: Uint8Array,
+  meta: PutMeta,
+  options: { partBytes?: number; concurrency?: number } = {},
+): Promise<void> {
+  const partBytes = options.partBytes ?? 8 * 1024 * 1024;
+  const concurrency = options.concurrency ?? 4;
+  if (partBytes < 5 * 1024 * 1024) throw new RangeError("Multipart parts must be at least 5 MiB");
+  if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32)
+    throw new RangeError("Multipart concurrency must be between 1 and 32");
+  const uploadId = await client.create(key, meta);
+  const chunks = Array.from({ length: Math.ceil(bytes.length / partBytes) }, (_, index) => ({
+    part: index + 1,
+    bytes: bytes.slice(index * partBytes, (index + 1) * partBytes),
+  }));
+  const completed: Array<{ part: number; etag: string }> = [];
+  try {
+    for (let offset = 0; offset < chunks.length; offset += concurrency) {
+      completed.push(
+        ...(await Promise.all(
+          chunks.slice(offset, offset + concurrency).map(async (chunk) => ({
+            part: chunk.part,
+            etag: await client.uploadPart(uploadId, key, chunk.part, chunk.bytes),
+          })),
+        )),
+      );
+    }
+    await client.complete(uploadId, key, completed);
+  } catch (error) {
+    await client.abort(uploadId, key);
+    throw error;
+  }
+}
+
+export interface TemporaryObject {
+  key: string;
+  expiresAt: number;
+}
+export function createTemporaryObjectCleaner(
+  driver: StorageDriver,
+  options: { now?: () => number; limit?: number } = {},
+) {
+  const objects = new Map<string, number>();
+  const now = options.now ?? Date.now;
+  const limit = options.limit ?? 10_000;
+  return {
+    track(key: string, ttlMs: number) {
+      if (objects.size >= limit) throw new Error("WRN-UPLOAD-TEMP-CAPACITY");
+      if (ttlMs < 1) throw new RangeError("Temporary TTL must be positive");
+      objects.set(key, now() + ttlMs);
+    },
+    async cleanup(at = now()) {
+      const due = [...objects].filter(([, expiry]) => expiry <= at);
+      for (const [key] of due) {
+        await driver.delete(key);
+        objects.delete(key);
+      }
+      return due.length;
+    },
+    snapshot: () => ({
+      tracked: objects.size,
+      nextExpiry: objects.size ? Math.min(...objects.values()) : undefined,
+    }),
+  };
+}
+
+export interface VideoTranscodeOptions {
+  format: "mp4" | "webm";
+  width?: number;
+  height?: number;
+  videoBitrateKbps?: number;
+}
+export function ffmpegVideoTranscoder(
+  options: { executable?: string; spawn?: (args: string[]) => { exited: Promise<number> } } = {},
+) {
+  return async (input: string, output: string, config: VideoTranscodeOptions): Promise<void> => {
+    if (!/^[\w .:\\/-]+$/.test(input) || !/^[\w .:\\/-]+$/.test(output))
+      throw new Error("Invalid video path");
+    const args = [
+      options.executable ?? "ffmpeg",
+      "-y",
+      "-i",
+      input,
+      ...(config.width || config.height
+        ? ["-vf", `scale=${config.width ?? -2}:${config.height ?? -2}`]
+        : []),
+      ...(config.videoBitrateKbps ? ["-b:v", `${config.videoBitrateKbps}k`] : []),
+      "-f",
+      config.format,
+      output,
+    ];
+    const child = options.spawn?.(args) ?? Bun.spawn(args, { stdout: "ignore", stderr: "ignore" });
+    if ((await child.exited) !== 0) throw new Error("WRN-VIDEO-TRANSCODE-FAILED");
+  };
+}

packages/uploader/src/plugin.ts

@@ -0,0 +1,20 @@
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { definePlugin } from "@wrnexus/plugin";
+export interface UploaderPluginOptions {
+  components?: boolean;
+  componentDir?: string;
+}
+const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
+export function uploaderComponentsDir(): string {
+  return join(packageRoot, "components");
+}
+export function uploaderPlugin(options: UploaderPluginOptions = {}) {
+  return definePlugin({
+    name: "@wrnexus/uploader",
+    version: "0.8.0",
+    componentDirs:
+      options.components === false ? [] : [options.componentDir ?? uploaderComponentsDir()],
+  });
+}
+export default uploaderPlugin;

packages/uploader/src/resumable.ts

@@ -0,0 +1,246 @@
+import type { StorageDriver } from "./driver.ts";
+import { accepts, contentTypeOf, extForType, extOf } from "./mime.ts";
+import { UploadError, type UploadedFile } from "./upload.ts";
+
+export interface ResumableUploadSession {
+  id: string;
+  key: string;
+  name: string;
+  type: string;
+  size: number;
+  chunkSize: number;
+  totalChunks: number;
+  createdAt: number;
+  expiresAt: number;
+  chunks: Record<number, Uint8Array>;
+  digests: Record<number, string>;
+}
+
+export interface ResumableSessionStore {
+  get(id: string): Promise<ResumableUploadSession | null>;
+  put(session: ResumableUploadSession): Promise<void>;
+  delete(id: string): Promise<void>;
+  list(): Promise<ResumableUploadSession[]>;
+}
+
+export function memoryResumableSessionStore(): ResumableSessionStore {
+  const sessions = new Map<string, ResumableUploadSession>();
+  return {
+    async get(id) {
+      return sessions.get(id) ?? null;
+    },
+    async put(session) {
+      sessions.set(session.id, session);
+    },
+    async delete(id) {
+      sessions.delete(id);
+    },
+    async list() {
+      return [...sessions.values()];
+    },
+  };
+}
+
+export interface ResumableUploadManagerOptions {
+  driver: StorageDriver;
+  sessions?: ResumableSessionStore;
+  maxBytes?: number;
+  chunkSize?: number;
+  maxSessions?: number;
+  ttlMs?: number;
+  accept?: string[];
+  prefix?: string;
+  publicUrl?: (key: string) => string | null;
+  now?: () => number;
+}
+
+export interface CreateResumableUpload {
+  name: string;
+  type?: string;
+  size: number;
+  chunkSize?: number;
+}
+
+export interface ResumableChunkResult {
+  receivedChunks: number;
+  totalChunks: number;
+  complete: boolean;
+  file?: UploadedFile;
+}
+
+export interface ResumableUploadManager {
+  create(input: CreateResumableUpload): Promise<ResumableUploadSession>;
+  uploadChunk(
+    id: string,
+    index: number,
+    data: Uint8Array,
+    sha256?: string,
+  ): Promise<ResumableChunkResult>;
+  status(
+    id: string,
+  ): Promise<{ received: number[]; totalChunks: number; expiresAt: number } | null>;
+  cancel(id: string): Promise<boolean>;
+  prune(): Promise<number>;
+}
+
+function safePrefix(prefix: string): string {
+  const value = prefix.replace(/^\/+|\/+$/g, "");
+  if (
+    value &&
+    (!/^[A-Za-z0-9._/-]+$/.test(value) ||
+      value.split("/").some((part) => !part || part === "." || part === ".."))
+  ) {
+    throw new UploadError("unsafe upload prefix", 400);
+  }
+  return value;
+}
+
+async function digest(data: Uint8Array): Promise<string> {
+  return Array.from(
+    new Uint8Array(await crypto.subtle.digest("SHA-256", data as BufferSource)),
+    (byte) => byte.toString(16).padStart(2, "0"),
+  ).join("");
+}
+
+export function createResumableUploadManager(
+  options: ResumableUploadManagerOptions,
+): ResumableUploadManager {
+  const sessions = options.sessions ?? memoryResumableSessionStore();
+  const maxBytes = options.maxBytes ?? 100 * 1024 * 1024;
+  const defaultChunkSize = options.chunkSize ?? 1024 * 1024;
+  const maxSessions = options.maxSessions ?? 1000;
+  const ttlMs = options.ttlMs ?? 24 * 60 * 60_000;
+  if (!Number.isInteger(maxBytes) || maxBytes < 1)
+    throw new RangeError("resumable maxBytes must be positive");
+  if (!Number.isInteger(defaultChunkSize) || defaultChunkSize < 1)
+    throw new RangeError("resumable chunkSize must be positive");
+  if (!Number.isInteger(maxSessions) || maxSessions < 1)
+    throw new RangeError("resumable maxSessions must be positive");
+  if (!Number.isFinite(ttlMs) || ttlMs <= 0)
+    throw new RangeError("resumable ttlMs must be positive");
+  const prefix = safePrefix(options.prefix ?? "resumable");
+  const now = options.now ?? Date.now;
+  const finalizing = new Map<string, Promise<UploadedFile>>();
+
+  const manager: ResumableUploadManager = {
+    async create(input) {
+      await manager.prune();
+      if (!Number.isInteger(input.size) || input.size < 1 || input.size > maxBytes)
+        throw new UploadError(`invalid upload size (max ${maxBytes} bytes)`, 413);
+      const type = input.type || contentTypeOf(input.name);
+      if (!accepts(options.accept, { type, name: input.name }))
+        throw new UploadError(`file type not allowed: ${type}`, 415);
+      if ((await sessions.list()).length >= maxSessions)
+        throw new UploadError("too many active resumable uploads", 429);
+      const chunkSize = input.chunkSize ?? defaultChunkSize;
+      if (!Number.isInteger(chunkSize) || chunkSize < 1 || chunkSize > maxBytes)
+        throw new UploadError("invalid resumable chunk size", 400);
+      const id = crypto.randomUUID();
+      const extension = extOf(input.name) || extForType(type);
+      const filename = extension ? `${id}.${extension}` : id;
+      const timestamp = now();
+      const session: ResumableUploadSession = {
+        id,
+        key: prefix ? `${prefix}/${filename}` : filename,
+        name: input.name.split(/[\\/]/).pop()?.slice(0, 255) || "file",
+        type,
+        size: input.size,
+        chunkSize,
+        totalChunks: Math.ceil(input.size / chunkSize),
+        createdAt: timestamp,
+        expiresAt: timestamp + ttlMs,
+        chunks: {},
+        digests: {},
+      };
+      await sessions.put(session);
+      return session;
+    },
+
+    async uploadChunk(id, index, data, expectedDigest) {
+      const session = await sessions.get(id);
+      if (!session || session.expiresAt <= now())
+        throw new UploadError("upload session not found or expired", 404);
+      if (!Number.isInteger(index) || index < 0 || index >= session.totalChunks)
+        throw new UploadError("invalid chunk index", 400);
+      const expectedSize =
+        index === session.totalChunks - 1
+          ? session.size - session.chunkSize * (session.totalChunks - 1)
+          : session.chunkSize;
+      if (data.byteLength !== expectedSize)
+        throw new UploadError(`invalid chunk size (expected ${expectedSize})`, 400);
+      const actualDigest = await digest(data);
+      if (expectedDigest && expectedDigest.toLowerCase() !== actualDigest)
+        throw new UploadError("chunk checksum mismatch", 422);
+      if (session.digests[index] && session.digests[index] !== actualDigest)
+        throw new UploadError("chunk already uploaded with different content", 409);
+      session.chunks[index] ??= data.slice();
+      session.digests[index] = actualDigest;
+      await sessions.put(session);
+      const receivedChunks = Object.keys(session.chunks).length;
+      if (receivedChunks !== session.totalChunks) {
+        return { receivedChunks, totalChunks: session.totalChunks, complete: false };
+      }
+      let completion = finalizing.get(id);
+      if (!completion) {
+        completion = (async () => {
+          const output = new Uint8Array(session.size);
+          let offset = 0;
+          for (let chunk = 0; chunk < session.totalChunks; chunk++) {
+            const value = session.chunks[chunk];
+            if (!value) throw new UploadError("upload is missing a chunk", 409);
+            output.set(value, offset);
+            offset += value.byteLength;
+          }
+          await options.driver.put(session.key, output, {
+            contentType: session.type,
+            filename: session.name,
+          });
+          await sessions.delete(id);
+          return {
+            key: session.key,
+            url: options.publicUrl?.(session.key) ?? null,
+            name: session.name,
+            type: session.type,
+            size: session.size,
+          };
+        })().finally(() => finalizing.delete(id));
+        finalizing.set(id, completion);
+      }
+      return {
+        receivedChunks,
+        totalChunks: session.totalChunks,
+        complete: true,
+        file: await completion,
+      };
+    },
+
+    async status(id) {
+      const session = await sessions.get(id);
+      return session
+        ? {
+            received: Object.keys(session.chunks)
+              .map(Number)
+              .sort((a, b) => a - b),
+            totalChunks: session.totalChunks,
+            expiresAt: session.expiresAt,
+          }
+        : null;
+    },
+    async cancel(id) {
+      if (!(await sessions.get(id))) return false;
+      await sessions.delete(id);
+      return true;
+    },
+    async prune() {
+      let removed = 0;
+      for (const session of await sessions.list()) {
+        if (session.expiresAt <= now()) {
+          await sessions.delete(session.id);
+          removed++;
+        }
+      }
+      return removed;
+    },
+  };
+  return manager;
+}

packages/uploader/src/upload.ts

@@ -31,6 +31,23 @@ export interface UploadOptions {
   accept?: string[];
   /** Key prefix, e.g. `"avatars"` → keys become `avatars/<yyyy>/<mm>/<rand>.<ext>`. */
   prefix?: string;
+  /** Virus/DLP/content scanner invoked before bytes enter storage. Throw or return unsafe to reject. */
+  scan?: (file: UploadScanInput) => UploadScanResult | Promise<UploadScanResult>;
+  /** Image/video/indexing hook invoked after storage. Failure removes the just-written object. */
+  afterStore?: (file: UploadedFile & { bytes: Uint8Array; store: Store }) => void | Promise<void>;
+}
+
+export interface UploadScanInput {
+  name: string;
+  type: string;
+  size: number;
+  bytes: Uint8Array;
+  store: Store;
+}
+export interface UploadScanResult {
+  safe: boolean;
+  reason?: string;
+  scanner?: string;
 }
 
 /** A 4xx-carrying error so `handleUpload` can map it to a status. */
@@ -123,14 +140,38 @@ export async function upload(
     }
     const key = makeKey({ name: file.name, type }, opts.prefix);
     const data = new Uint8Array(await file.arrayBuffer());
+    if (opts.scan) {
+      const result = await opts.scan({
+        name: displayName(file.name),
+        type,
+        size: file.size,
+        bytes: data,
+        store,
+      });
+      if (!result.safe)
+        throw new UploadError(
+          `file rejected by ${result.scanner ?? "security scanner"}${result.reason ? `: ${result.reason}` : ""}`,
+          422,
+        );
+    }
     await store.driver.put(key, data, { contentType: type, filename: displayName(file.name) });
-    files.push({
+    const uploaded: UploadedFile = {
       key,
       url: storedUrl(store, key),
       name: displayName(file.name),
       type,
       size: file.size,
-    });
+    };
+    try {
+      await opts.afterStore?.({ ...uploaded, bytes: data, store });
+    } catch (error) {
+      await store.driver.delete(key);
+      throw new UploadError(
+        `post-upload processing failed: ${error instanceof Error ? error.message : String(error)}`,
+        422,
+      );
+    }
+    files.push(uploaded);
   }
   return { files };
 }