release: WRNexusJS 0.8.0
This commit is contained in:
@@ -108,6 +108,11 @@ export const GET = serveFromStore("docs"); // your middleware decides who gets i
|
||||
|
||||
## API
|
||||
|
||||
Uploads can participate in security and media pipelines without changing storage drivers. Pass a
|
||||
`scan` hook to reject malware/DLP findings before storage, and `afterStore` to enqueue image/video
|
||||
processing or indexing. If post-processing throws, WRNexus deletes the newly written object so a
|
||||
partially accepted upload is never left behind.
|
||||
|
||||
| Export | What |
|
||||
| --------------------------------------- | --------------------------------------------------------------- |
|
||||
| `handleUpload(opts)` | POST route handler → JSON `{ ok, files }` |
|
||||
@@ -123,3 +128,36 @@ export const GET = serveFromStore("docs"); // your middleware decides who gets i
|
||||
- SigV4 signing is implemented from scratch (no `@aws-sdk`); tested against local S3 semantics.
|
||||
Live AWS/R2 connectivity depends on your credentials + bucket policy.
|
||||
- v1 buffers each file in memory up to its size cap (fine for images/docs up to tens of MB).
|
||||
|
||||
## Helper and component kit
|
||||
|
||||
Use `formatFileSize`, `uploadAccept`, `uploadedFileMap`, `uploaderAttributes`, and `assertUploadedFiles` to keep upload forms and server validation consistent.
|
||||
|
||||
Enable `uploaderPlugin()` for:
|
||||
|
||||
- `<UploadDropzone />`
|
||||
- `<UploadStatus />`
|
||||
|
||||
The complete blocks compose `Card`, `Alert`, and `Badge` from `@wrnexus/ui`; the specialized upload runtime remains responsible for the native file input and secure transport behavior.
|
||||
Large files can use `createResumableUploadManager`. Sessions are bounded and
|
||||
expiring; chunks may arrive out of order, carry SHA-256 checksums, and are
|
||||
idempotent when retried. Conflicting retries reject, and the object is assembled
|
||||
only after every exact-sized chunk is present.
|
||||
|
||||
```ts
|
||||
const uploads = createResumableUploadManager({
|
||||
driver: getStore("documents").driver,
|
||||
sessions: redisUploadSessionStore,
|
||||
chunkSize: 5 * 1024 * 1024,
|
||||
maxBytes: 500 * 1024 * 1024,
|
||||
accept: ["application/pdf"],
|
||||
});
|
||||
|
||||
const session = await uploads.create({ name: "report.pdf", size, type });
|
||||
await uploads.uploadChunk(session.id, index, bytes, sha256);
|
||||
```
|
||||
|
||||
The included memory session store is intended for one-process apps and tests.
|
||||
Multi-instance production deployments should implement `ResumableSessionStore`
|
||||
with shared durable storage and atomic session updates, and periodically call
|
||||
`prune()` for abandoned uploads.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
component UploadDropzone {
|
||||
props {
|
||||
store: string = "default"
|
||||
endpoint: string = "/api/upload"
|
||||
accept: string = ""
|
||||
maxBytes: number = 0
|
||||
multiple: boolean = false
|
||||
field: string = "file"
|
||||
label: string = "Drag files here or click to browse"
|
||||
title: string = "Upload files"
|
||||
description: string = "Files are validated and uploaded securely."
|
||||
color: string = "primary"
|
||||
size: string = "md"
|
||||
class: string = ""
|
||||
}
|
||||
view {
|
||||
<Card {...attrs} title='{title}' description='{description}' color='{color}' size='{size}' class='{class}'>
|
||||
<div data-uploader='{store}' data-endpoint='{endpoint}' data-accept='{accept}' data-max='{maxBytes}' data-multiple='{multiple}' data-field='{field}' data-label='{label}'></div>
|
||||
</Card>
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
component UploadStatus {
|
||||
props {
|
||||
files: unknown[] = []
|
||||
title: string = "Uploaded files"
|
||||
emptyMessage: string = "No files uploaded yet."
|
||||
color: string = "primary"
|
||||
size: string = "sm"
|
||||
class: string = ""
|
||||
}
|
||||
view {
|
||||
<Card {...attrs} title='{title}' color='{color}' size='{size}' class='{class}'>
|
||||
{#if files.length == 0}<Alert title="Waiting for files" description='{emptyMessage}' color="info" variant="soft" size='{size}' />{/if}
|
||||
<div class="space-y-2">
|
||||
{#each files as file}
|
||||
<div class="flex items-center gap-3 rounded-[var(--wire-radius-sm)] border border-[var(--wire-color-border)] p-3">
|
||||
<span class="icon-[lucide--file-check-2] size-5 text-[var(--wire-color-success)]"></span>
|
||||
<div class="min-w-0 flex-1"><p class="m-0 truncate text-sm font-semibold">{file.name}</p><p class="m-0 text-xs text-[var(--wire-color-muted)]">{file.type || "File"} · {file.size || 0} bytes</p></div>
|
||||
<Badge label='{file.status || "uploaded"}' color='{file.status == "error" ? "danger" : "success"}' variant="soft" size='{size}' />
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
</Card>
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,36 @@
|
||||
{
|
||||
"name": "@wrnexus/uploader",
|
||||
"version": "0.7.0",
|
||||
"version": "0.8.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "src/index.ts",
|
||||
"main": "./src/index.ts",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
".": "./src/index.ts",
|
||||
"./plugin": "./src/plugin.ts",
|
||||
"./components/*": "./components/*"
|
||||
},
|
||||
"dependencies": {
|
||||
"@wrnexus/core": "workspace:*"
|
||||
"@wrnexus/core": "workspace:*",
|
||||
"@wrnexus/plugin": "workspace:*",
|
||||
"@wrnexus/ui": "workspace:*"
|
||||
},
|
||||
"description": "Secure upload drivers, policies, client runtime, helper functions, and reusable upload components.",
|
||||
"types": "./src/index.ts",
|
||||
"files": [
|
||||
"src",
|
||||
"components",
|
||||
"README.md"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.3.14",
|
||||
"typescript": "^5.9.2",
|
||||
"@wrnexus/syntax": "workspace:*"
|
||||
},
|
||||
"wrnexus": {
|
||||
"plugin": {
|
||||
"plugin": "./src/plugin.ts",
|
||||
"export": "default",
|
||||
"factory": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { UploadedFile } from "./upload.ts";
|
||||
|
||||
export function formatFileSize(bytes: number, locale = "en"): string {
|
||||
const value = Math.max(0, Number(bytes) || 0);
|
||||
if (value < 1_024) return `${value} B`;
|
||||
const units = ["KB", "MB", "GB", "TB"];
|
||||
let current = value / 1_024;
|
||||
let unit = units[0]!;
|
||||
for (let index = 1; index < units.length && current >= 1_024; index++) {
|
||||
current /= 1_024;
|
||||
unit = units[index]!;
|
||||
}
|
||||
return `${new Intl.NumberFormat(locale, { maximumFractionDigits: current < 10 ? 1 : 0 }).format(current)} ${unit}`;
|
||||
}
|
||||
|
||||
export function uploadAccept(value: string | readonly string[]): string {
|
||||
return (typeof value === "string" ? value.split(",") : [...value])
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
.join(",");
|
||||
}
|
||||
|
||||
export function uploadedFileMap(files: readonly UploadedFile[]): Record<string, UploadedFile> {
|
||||
return Object.fromEntries(files.map((file) => [file.key, file]));
|
||||
}
|
||||
|
||||
export function uploaderAttributes(
|
||||
options: {
|
||||
store?: string;
|
||||
endpoint?: string;
|
||||
accept?: string | readonly string[];
|
||||
maxBytes?: number;
|
||||
multiple?: boolean;
|
||||
field?: string;
|
||||
label?: string;
|
||||
} = {},
|
||||
): Record<string, string | boolean> {
|
||||
return {
|
||||
"data-uploader": options.store ?? "default",
|
||||
"data-endpoint": options.endpoint ?? "/api/upload",
|
||||
...(options.accept ? { "data-accept": uploadAccept(options.accept) } : {}),
|
||||
...(options.maxBytes ? { "data-max": String(options.maxBytes) } : {}),
|
||||
...(options.multiple ? { "data-multiple": true } : {}),
|
||||
...(options.field ? { "data-field": options.field } : {}),
|
||||
...(options.label ? { "data-label": options.label } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function assertUploadedFiles(
|
||||
files: readonly UploadedFile[],
|
||||
options: { min?: number; max?: number } = {},
|
||||
): readonly UploadedFile[] {
|
||||
const min = Math.max(0, options.min ?? 0);
|
||||
const max = Math.max(min, options.max ?? Number.POSITIVE_INFINITY);
|
||||
if (files.length < min) throw new Error(`WRN-UPLOAD-MIN-FILES: expected at least ${min}`);
|
||||
if (files.length > max) throw new Error(`WRN-UPLOAD-MAX-FILES: expected at most ${max}`);
|
||||
return files;
|
||||
}
|
||||
@@ -38,7 +38,7 @@ export {
|
||||
UploadError,
|
||||
UPLOADS_PREFIX,
|
||||
} from "./upload.ts";
|
||||
export type { UploadedFile, UploadOptions } from "./upload.ts";
|
||||
export type { UploadedFile, UploadOptions, UploadScanInput, UploadScanResult } from "./upload.ts";
|
||||
|
||||
export { configureStorage, getStore, hasStorage, storeNames } from "./client.ts";
|
||||
export type { Store } from "./client.ts";
|
||||
@@ -70,3 +70,37 @@ export {
|
||||
UploadPolicyError,
|
||||
} from "./security.ts";
|
||||
export type { UploadPolicy, UploadInspection, SignedFileToken } from "./security.ts";
|
||||
export {
|
||||
formatFileSize,
|
||||
uploadAccept,
|
||||
uploadedFileMap,
|
||||
uploaderAttributes,
|
||||
assertUploadedFiles,
|
||||
} from "./helpers.ts";
|
||||
export { uploaderPlugin, uploaderComponentsDir } from "./plugin.ts";
|
||||
export type { UploaderPluginOptions } from "./plugin.ts";
|
||||
export { createResumableUploadManager, memoryResumableSessionStore } from "./resumable.ts";
|
||||
export type {
|
||||
ResumableUploadManager,
|
||||
ResumableUploadManagerOptions,
|
||||
ResumableUploadSession,
|
||||
ResumableSessionStore,
|
||||
CreateResumableUpload,
|
||||
ResumableChunkResult,
|
||||
} from "./resumable.ts";
|
||||
export {
|
||||
memoryQuotaStore,
|
||||
postgresQuotaStore,
|
||||
POSTGRES_QUOTA_SCHEMA,
|
||||
multipartUpload,
|
||||
createTemporaryObjectCleaner,
|
||||
ffmpegVideoTranscoder,
|
||||
} from "./operations.ts";
|
||||
export type {
|
||||
QuotaUsage,
|
||||
QuotaStore,
|
||||
QuotaSqlClient,
|
||||
MultipartObjectClient,
|
||||
TemporaryObject,
|
||||
VideoTranscodeOptions,
|
||||
} from "./operations.ts";
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
import type { PutMeta, StorageDriver } from "./driver.ts";
|
||||
|
||||
export interface QuotaUsage {
|
||||
owner: string;
|
||||
bytes: number;
|
||||
objects: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
export interface QuotaStore {
|
||||
get(owner: string): Promise<QuotaUsage>;
|
||||
reserve(
|
||||
owner: string,
|
||||
bytes: number,
|
||||
limits: { bytes: number; objects?: number },
|
||||
): Promise<boolean>;
|
||||
release(owner: string, bytes: number): Promise<void>;
|
||||
}
|
||||
|
||||
export function memoryQuotaStore(): QuotaStore {
|
||||
const values = new Map<string, QuotaUsage>();
|
||||
return {
|
||||
async get(owner) {
|
||||
return structuredClone(
|
||||
values.get(owner) ?? { owner, bytes: 0, objects: 0, updatedAt: Date.now() },
|
||||
);
|
||||
},
|
||||
async reserve(owner, bytes, limits) {
|
||||
if (!Number.isInteger(bytes) || bytes < 0)
|
||||
throw new RangeError("Quota bytes must be non-negative");
|
||||
const current = values.get(owner) ?? { owner, bytes: 0, objects: 0, updatedAt: Date.now() };
|
||||
if (
|
||||
current.bytes + bytes > limits.bytes ||
|
||||
current.objects + 1 > (limits.objects ?? Number.MAX_SAFE_INTEGER)
|
||||
)
|
||||
return false;
|
||||
values.set(owner, {
|
||||
owner,
|
||||
bytes: current.bytes + bytes,
|
||||
objects: current.objects + 1,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
return true;
|
||||
},
|
||||
async release(owner, bytes) {
|
||||
const current = values.get(owner);
|
||||
if (!current) return;
|
||||
values.set(owner, {
|
||||
...current,
|
||||
bytes: Math.max(0, current.bytes - Math.max(0, bytes)),
|
||||
objects: Math.max(0, current.objects - 1),
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface QuotaSqlClient {
|
||||
query<T = any>(sql: string, parameters?: unknown[]): Promise<{ rows: T[] }>;
|
||||
}
|
||||
|
||||
/** PostgreSQL quota accounting using a single atomic conditional upsert. */
|
||||
export function postgresQuotaStore(
|
||||
db: QuotaSqlClient,
|
||||
table = "wrnexus_storage_quota",
|
||||
): QuotaStore {
|
||||
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) throw new Error("Invalid quota table name");
|
||||
return {
|
||||
async get(owner) {
|
||||
const result = await db.query<QuotaUsage>(
|
||||
`SELECT owner,bytes,objects,updated_at AS "updatedAt" FROM ${table} WHERE owner=$1`,
|
||||
[owner],
|
||||
);
|
||||
return result.rows[0] ?? { owner, bytes: 0, objects: 0, updatedAt: Date.now() };
|
||||
},
|
||||
async reserve(owner, bytes, limits) {
|
||||
const result = await db.query(
|
||||
`INSERT INTO ${table} (owner,bytes,objects,updated_at) VALUES ($1,$2,1,$5) ON CONFLICT (owner) DO UPDATE SET bytes=${table}.bytes+$2,objects=${table}.objects+1,updated_at=$5 WHERE ${table}.bytes+$2 <= $3 AND ${table}.objects+1 <= $4 RETURNING owner`,
|
||||
[owner, bytes, limits.bytes, limits.objects ?? 2_147_483_647, Date.now()],
|
||||
);
|
||||
return result.rows.length === 1;
|
||||
},
|
||||
async release(owner, bytes) {
|
||||
await db.query(
|
||||
`UPDATE ${table} SET bytes=GREATEST(0,bytes-$2),objects=GREATEST(0,objects-1),updated_at=$3 WHERE owner=$1`,
|
||||
[owner, bytes, Date.now()],
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const POSTGRES_QUOTA_SCHEMA = `CREATE TABLE IF NOT EXISTS wrnexus_storage_quota (owner text PRIMARY KEY, bytes bigint NOT NULL DEFAULT 0, objects integer NOT NULL DEFAULT 0, updated_at bigint NOT NULL);`;
|
||||
|
||||
export interface MultipartObjectClient {
|
||||
create(key: string, meta: PutMeta): Promise<string>;
|
||||
uploadPart(uploadId: string, key: string, part: number, bytes: Uint8Array): Promise<string>;
|
||||
complete(
|
||||
uploadId: string,
|
||||
key: string,
|
||||
parts: Array<{ part: number; etag: string }>,
|
||||
): Promise<void>;
|
||||
abort(uploadId: string, key: string): Promise<void>;
|
||||
}
|
||||
|
||||
export async function multipartUpload(
|
||||
client: MultipartObjectClient,
|
||||
key: string,
|
||||
bytes: Uint8Array,
|
||||
meta: PutMeta,
|
||||
options: { partBytes?: number; concurrency?: number } = {},
|
||||
): Promise<void> {
|
||||
const partBytes = options.partBytes ?? 8 * 1024 * 1024;
|
||||
const concurrency = options.concurrency ?? 4;
|
||||
if (partBytes < 5 * 1024 * 1024) throw new RangeError("Multipart parts must be at least 5 MiB");
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32)
|
||||
throw new RangeError("Multipart concurrency must be between 1 and 32");
|
||||
const uploadId = await client.create(key, meta);
|
||||
const chunks = Array.from({ length: Math.ceil(bytes.length / partBytes) }, (_, index) => ({
|
||||
part: index + 1,
|
||||
bytes: bytes.slice(index * partBytes, (index + 1) * partBytes),
|
||||
}));
|
||||
const completed: Array<{ part: number; etag: string }> = [];
|
||||
try {
|
||||
for (let offset = 0; offset < chunks.length; offset += concurrency) {
|
||||
completed.push(
|
||||
...(await Promise.all(
|
||||
chunks.slice(offset, offset + concurrency).map(async (chunk) => ({
|
||||
part: chunk.part,
|
||||
etag: await client.uploadPart(uploadId, key, chunk.part, chunk.bytes),
|
||||
})),
|
||||
)),
|
||||
);
|
||||
}
|
||||
await client.complete(uploadId, key, completed);
|
||||
} catch (error) {
|
||||
await client.abort(uploadId, key);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export interface TemporaryObject {
|
||||
key: string;
|
||||
expiresAt: number;
|
||||
}
|
||||
export function createTemporaryObjectCleaner(
|
||||
driver: StorageDriver,
|
||||
options: { now?: () => number; limit?: number } = {},
|
||||
) {
|
||||
const objects = new Map<string, number>();
|
||||
const now = options.now ?? Date.now;
|
||||
const limit = options.limit ?? 10_000;
|
||||
return {
|
||||
track(key: string, ttlMs: number) {
|
||||
if (objects.size >= limit) throw new Error("WRN-UPLOAD-TEMP-CAPACITY");
|
||||
if (ttlMs < 1) throw new RangeError("Temporary TTL must be positive");
|
||||
objects.set(key, now() + ttlMs);
|
||||
},
|
||||
async cleanup(at = now()) {
|
||||
const due = [...objects].filter(([, expiry]) => expiry <= at);
|
||||
for (const [key] of due) {
|
||||
await driver.delete(key);
|
||||
objects.delete(key);
|
||||
}
|
||||
return due.length;
|
||||
},
|
||||
snapshot: () => ({
|
||||
tracked: objects.size,
|
||||
nextExpiry: objects.size ? Math.min(...objects.values()) : undefined,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface VideoTranscodeOptions {
|
||||
format: "mp4" | "webm";
|
||||
width?: number;
|
||||
height?: number;
|
||||
videoBitrateKbps?: number;
|
||||
}
|
||||
export function ffmpegVideoTranscoder(
|
||||
options: { executable?: string; spawn?: (args: string[]) => { exited: Promise<number> } } = {},
|
||||
) {
|
||||
return async (input: string, output: string, config: VideoTranscodeOptions): Promise<void> => {
|
||||
if (!/^[\w .:\\/-]+$/.test(input) || !/^[\w .:\\/-]+$/.test(output))
|
||||
throw new Error("Invalid video path");
|
||||
const args = [
|
||||
options.executable ?? "ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
input,
|
||||
...(config.width || config.height
|
||||
? ["-vf", `scale=${config.width ?? -2}:${config.height ?? -2}`]
|
||||
: []),
|
||||
...(config.videoBitrateKbps ? ["-b:v", `${config.videoBitrateKbps}k`] : []),
|
||||
"-f",
|
||||
config.format,
|
||||
output,
|
||||
];
|
||||
const child = options.spawn?.(args) ?? Bun.spawn(args, { stdout: "ignore", stderr: "ignore" });
|
||||
if ((await child.exited) !== 0) throw new Error("WRN-VIDEO-TRANSCODE-FAILED");
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { definePlugin } from "@wrnexus/plugin";
|
||||
export interface UploaderPluginOptions {
|
||||
components?: boolean;
|
||||
componentDir?: string;
|
||||
}
|
||||
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
export function uploaderComponentsDir(): string {
|
||||
return join(packageRoot, "components");
|
||||
}
|
||||
export function uploaderPlugin(options: UploaderPluginOptions = {}) {
|
||||
return definePlugin({
|
||||
name: "@wrnexus/uploader",
|
||||
version: "0.8.0",
|
||||
componentDirs:
|
||||
options.components === false ? [] : [options.componentDir ?? uploaderComponentsDir()],
|
||||
});
|
||||
}
|
||||
export default uploaderPlugin;
|
||||
@@ -0,0 +1,246 @@
|
||||
import type { StorageDriver } from "./driver.ts";
|
||||
import { accepts, contentTypeOf, extForType, extOf } from "./mime.ts";
|
||||
import { UploadError, type UploadedFile } from "./upload.ts";
|
||||
|
||||
export interface ResumableUploadSession {
|
||||
id: string;
|
||||
key: string;
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
chunkSize: number;
|
||||
totalChunks: number;
|
||||
createdAt: number;
|
||||
expiresAt: number;
|
||||
chunks: Record<number, Uint8Array>;
|
||||
digests: Record<number, string>;
|
||||
}
|
||||
|
||||
export interface ResumableSessionStore {
|
||||
get(id: string): Promise<ResumableUploadSession | null>;
|
||||
put(session: ResumableUploadSession): Promise<void>;
|
||||
delete(id: string): Promise<void>;
|
||||
list(): Promise<ResumableUploadSession[]>;
|
||||
}
|
||||
|
||||
export function memoryResumableSessionStore(): ResumableSessionStore {
|
||||
const sessions = new Map<string, ResumableUploadSession>();
|
||||
return {
|
||||
async get(id) {
|
||||
return sessions.get(id) ?? null;
|
||||
},
|
||||
async put(session) {
|
||||
sessions.set(session.id, session);
|
||||
},
|
||||
async delete(id) {
|
||||
sessions.delete(id);
|
||||
},
|
||||
async list() {
|
||||
return [...sessions.values()];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface ResumableUploadManagerOptions {
|
||||
driver: StorageDriver;
|
||||
sessions?: ResumableSessionStore;
|
||||
maxBytes?: number;
|
||||
chunkSize?: number;
|
||||
maxSessions?: number;
|
||||
ttlMs?: number;
|
||||
accept?: string[];
|
||||
prefix?: string;
|
||||
publicUrl?: (key: string) => string | null;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export interface CreateResumableUpload {
|
||||
name: string;
|
||||
type?: string;
|
||||
size: number;
|
||||
chunkSize?: number;
|
||||
}
|
||||
|
||||
export interface ResumableChunkResult {
|
||||
receivedChunks: number;
|
||||
totalChunks: number;
|
||||
complete: boolean;
|
||||
file?: UploadedFile;
|
||||
}
|
||||
|
||||
export interface ResumableUploadManager {
|
||||
create(input: CreateResumableUpload): Promise<ResumableUploadSession>;
|
||||
uploadChunk(
|
||||
id: string,
|
||||
index: number,
|
||||
data: Uint8Array,
|
||||
sha256?: string,
|
||||
): Promise<ResumableChunkResult>;
|
||||
status(
|
||||
id: string,
|
||||
): Promise<{ received: number[]; totalChunks: number; expiresAt: number } | null>;
|
||||
cancel(id: string): Promise<boolean>;
|
||||
prune(): Promise<number>;
|
||||
}
|
||||
|
||||
function safePrefix(prefix: string): string {
|
||||
const value = prefix.replace(/^\/+|\/+$/g, "");
|
||||
if (
|
||||
value &&
|
||||
(!/^[A-Za-z0-9._/-]+$/.test(value) ||
|
||||
value.split("/").some((part) => !part || part === "." || part === ".."))
|
||||
) {
|
||||
throw new UploadError("unsafe upload prefix", 400);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function digest(data: Uint8Array): Promise<string> {
|
||||
return Array.from(
|
||||
new Uint8Array(await crypto.subtle.digest("SHA-256", data as BufferSource)),
|
||||
(byte) => byte.toString(16).padStart(2, "0"),
|
||||
).join("");
|
||||
}
|
||||
|
||||
export function createResumableUploadManager(
|
||||
options: ResumableUploadManagerOptions,
|
||||
): ResumableUploadManager {
|
||||
const sessions = options.sessions ?? memoryResumableSessionStore();
|
||||
const maxBytes = options.maxBytes ?? 100 * 1024 * 1024;
|
||||
const defaultChunkSize = options.chunkSize ?? 1024 * 1024;
|
||||
const maxSessions = options.maxSessions ?? 1000;
|
||||
const ttlMs = options.ttlMs ?? 24 * 60 * 60_000;
|
||||
if (!Number.isInteger(maxBytes) || maxBytes < 1)
|
||||
throw new RangeError("resumable maxBytes must be positive");
|
||||
if (!Number.isInteger(defaultChunkSize) || defaultChunkSize < 1)
|
||||
throw new RangeError("resumable chunkSize must be positive");
|
||||
if (!Number.isInteger(maxSessions) || maxSessions < 1)
|
||||
throw new RangeError("resumable maxSessions must be positive");
|
||||
if (!Number.isFinite(ttlMs) || ttlMs <= 0)
|
||||
throw new RangeError("resumable ttlMs must be positive");
|
||||
const prefix = safePrefix(options.prefix ?? "resumable");
|
||||
const now = options.now ?? Date.now;
|
||||
const finalizing = new Map<string, Promise<UploadedFile>>();
|
||||
|
||||
const manager: ResumableUploadManager = {
|
||||
async create(input) {
|
||||
await manager.prune();
|
||||
if (!Number.isInteger(input.size) || input.size < 1 || input.size > maxBytes)
|
||||
throw new UploadError(`invalid upload size (max ${maxBytes} bytes)`, 413);
|
||||
const type = input.type || contentTypeOf(input.name);
|
||||
if (!accepts(options.accept, { type, name: input.name }))
|
||||
throw new UploadError(`file type not allowed: ${type}`, 415);
|
||||
if ((await sessions.list()).length >= maxSessions)
|
||||
throw new UploadError("too many active resumable uploads", 429);
|
||||
const chunkSize = input.chunkSize ?? defaultChunkSize;
|
||||
if (!Number.isInteger(chunkSize) || chunkSize < 1 || chunkSize > maxBytes)
|
||||
throw new UploadError("invalid resumable chunk size", 400);
|
||||
const id = crypto.randomUUID();
|
||||
const extension = extOf(input.name) || extForType(type);
|
||||
const filename = extension ? `${id}.${extension}` : id;
|
||||
const timestamp = now();
|
||||
const session: ResumableUploadSession = {
|
||||
id,
|
||||
key: prefix ? `${prefix}/${filename}` : filename,
|
||||
name: input.name.split(/[\\/]/).pop()?.slice(0, 255) || "file",
|
||||
type,
|
||||
size: input.size,
|
||||
chunkSize,
|
||||
totalChunks: Math.ceil(input.size / chunkSize),
|
||||
createdAt: timestamp,
|
||||
expiresAt: timestamp + ttlMs,
|
||||
chunks: {},
|
||||
digests: {},
|
||||
};
|
||||
await sessions.put(session);
|
||||
return session;
|
||||
},
|
||||
|
||||
async uploadChunk(id, index, data, expectedDigest) {
|
||||
const session = await sessions.get(id);
|
||||
if (!session || session.expiresAt <= now())
|
||||
throw new UploadError("upload session not found or expired", 404);
|
||||
if (!Number.isInteger(index) || index < 0 || index >= session.totalChunks)
|
||||
throw new UploadError("invalid chunk index", 400);
|
||||
const expectedSize =
|
||||
index === session.totalChunks - 1
|
||||
? session.size - session.chunkSize * (session.totalChunks - 1)
|
||||
: session.chunkSize;
|
||||
if (data.byteLength !== expectedSize)
|
||||
throw new UploadError(`invalid chunk size (expected ${expectedSize})`, 400);
|
||||
const actualDigest = await digest(data);
|
||||
if (expectedDigest && expectedDigest.toLowerCase() !== actualDigest)
|
||||
throw new UploadError("chunk checksum mismatch", 422);
|
||||
if (session.digests[index] && session.digests[index] !== actualDigest)
|
||||
throw new UploadError("chunk already uploaded with different content", 409);
|
||||
session.chunks[index] ??= data.slice();
|
||||
session.digests[index] = actualDigest;
|
||||
await sessions.put(session);
|
||||
const receivedChunks = Object.keys(session.chunks).length;
|
||||
if (receivedChunks !== session.totalChunks) {
|
||||
return { receivedChunks, totalChunks: session.totalChunks, complete: false };
|
||||
}
|
||||
let completion = finalizing.get(id);
|
||||
if (!completion) {
|
||||
completion = (async () => {
|
||||
const output = new Uint8Array(session.size);
|
||||
let offset = 0;
|
||||
for (let chunk = 0; chunk < session.totalChunks; chunk++) {
|
||||
const value = session.chunks[chunk];
|
||||
if (!value) throw new UploadError("upload is missing a chunk", 409);
|
||||
output.set(value, offset);
|
||||
offset += value.byteLength;
|
||||
}
|
||||
await options.driver.put(session.key, output, {
|
||||
contentType: session.type,
|
||||
filename: session.name,
|
||||
});
|
||||
await sessions.delete(id);
|
||||
return {
|
||||
key: session.key,
|
||||
url: options.publicUrl?.(session.key) ?? null,
|
||||
name: session.name,
|
||||
type: session.type,
|
||||
size: session.size,
|
||||
};
|
||||
})().finally(() => finalizing.delete(id));
|
||||
finalizing.set(id, completion);
|
||||
}
|
||||
return {
|
||||
receivedChunks,
|
||||
totalChunks: session.totalChunks,
|
||||
complete: true,
|
||||
file: await completion,
|
||||
};
|
||||
},
|
||||
|
||||
async status(id) {
|
||||
const session = await sessions.get(id);
|
||||
return session
|
||||
? {
|
||||
received: Object.keys(session.chunks)
|
||||
.map(Number)
|
||||
.sort((a, b) => a - b),
|
||||
totalChunks: session.totalChunks,
|
||||
expiresAt: session.expiresAt,
|
||||
}
|
||||
: null;
|
||||
},
|
||||
async cancel(id) {
|
||||
if (!(await sessions.get(id))) return false;
|
||||
await sessions.delete(id);
|
||||
return true;
|
||||
},
|
||||
async prune() {
|
||||
let removed = 0;
|
||||
for (const session of await sessions.list()) {
|
||||
if (session.expiresAt <= now()) {
|
||||
await sessions.delete(session.id);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
},
|
||||
};
|
||||
return manager;
|
||||
}
|
||||
@@ -31,6 +31,23 @@ export interface UploadOptions {
|
||||
accept?: string[];
|
||||
/** Key prefix, e.g. `"avatars"` → keys become `avatars/<yyyy>/<mm>/<rand>.<ext>`. */
|
||||
prefix?: string;
|
||||
/** Virus/DLP/content scanner invoked before bytes enter storage. Throw or return unsafe to reject. */
|
||||
scan?: (file: UploadScanInput) => UploadScanResult | Promise<UploadScanResult>;
|
||||
/** Image/video/indexing hook invoked after storage. Failure removes the just-written object. */
|
||||
afterStore?: (file: UploadedFile & { bytes: Uint8Array; store: Store }) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface UploadScanInput {
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
bytes: Uint8Array;
|
||||
store: Store;
|
||||
}
|
||||
export interface UploadScanResult {
|
||||
safe: boolean;
|
||||
reason?: string;
|
||||
scanner?: string;
|
||||
}
|
||||
|
||||
/** A 4xx-carrying error so `handleUpload` can map it to a status. */
|
||||
@@ -123,14 +140,38 @@ export async function upload(
|
||||
}
|
||||
const key = makeKey({ name: file.name, type }, opts.prefix);
|
||||
const data = new Uint8Array(await file.arrayBuffer());
|
||||
if (opts.scan) {
|
||||
const result = await opts.scan({
|
||||
name: displayName(file.name),
|
||||
type,
|
||||
size: file.size,
|
||||
bytes: data,
|
||||
store,
|
||||
});
|
||||
if (!result.safe)
|
||||
throw new UploadError(
|
||||
`file rejected by ${result.scanner ?? "security scanner"}${result.reason ? `: ${result.reason}` : ""}`,
|
||||
422,
|
||||
);
|
||||
}
|
||||
await store.driver.put(key, data, { contentType: type, filename: displayName(file.name) });
|
||||
files.push({
|
||||
const uploaded: UploadedFile = {
|
||||
key,
|
||||
url: storedUrl(store, key),
|
||||
name: displayName(file.name),
|
||||
type,
|
||||
size: file.size,
|
||||
});
|
||||
};
|
||||
try {
|
||||
await opts.afterStore?.({ ...uploaded, bytes: data, store });
|
||||
} catch (error) {
|
||||
await store.driver.delete(key);
|
||||
throw new UploadError(
|
||||
`post-upload processing failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
422,
|
||||
);
|
||||
}
|
||||
files.push(uploaded);
|
||||
}
|
||||
return { files };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
assertUploadedFiles,
|
||||
formatFileSize,
|
||||
uploadAccept,
|
||||
uploaderAttributes,
|
||||
} from "../src/index.ts";
|
||||
|
||||
describe("uploader helper kit", () => {
|
||||
test("formats sizes and builds uploader attributes", () => {
|
||||
expect(formatFileSize(1024)).toContain("KB");
|
||||
expect(uploadAccept(["image/png", ".jpg"])).toBe("image/png,.jpg");
|
||||
expect(uploaderAttributes({ store: "public", multiple: true })["data-uploader"]).toBe("public");
|
||||
});
|
||||
|
||||
test("asserts uploaded file count and status", () => {
|
||||
const files = [{ name: "a.png", size: 10, type: "image/png", key: "a", url: "/a" }];
|
||||
expect(assertUploadedFiles(files, { min: 1, max: 1 })).toEqual(files);
|
||||
expect(() => assertUploadedFiles([], { min: 1 })).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import {
|
||||
createTemporaryObjectCleaner,
|
||||
ffmpegVideoTranscoder,
|
||||
memoryQuotaStore,
|
||||
multipartUpload,
|
||||
postgresQuotaStore,
|
||||
} from "../src/index.ts";
|
||||
|
||||
test("quota stores enforce durable byte and object limits", async () => {
|
||||
const quota = memoryQuotaStore();
|
||||
expect(await quota.reserve("tenant", 60, { bytes: 100, objects: 2 })).toBeTrue();
|
||||
expect(await quota.reserve("tenant", 50, { bytes: 100, objects: 2 })).toBeFalse();
|
||||
await quota.release("tenant", 60);
|
||||
expect(await quota.get("tenant")).toMatchObject({ bytes: 0, objects: 0 });
|
||||
const calls: unknown[][] = [];
|
||||
const sql = postgresQuotaStore({
|
||||
async query<T>(_sql: string, parameters?: unknown[]) {
|
||||
calls.push(parameters ?? []);
|
||||
return { rows: [{ owner: "tenant" } as T] };
|
||||
},
|
||||
});
|
||||
expect(await sql.reserve("tenant", 10, { bytes: 20 })).toBeTrue();
|
||||
expect(calls[0]?.[0]).toBe("tenant");
|
||||
});
|
||||
|
||||
test("multipart uploader limits concurrency, completes and aborts failures", async () => {
|
||||
const uploaded: number[] = [];
|
||||
let completed = false;
|
||||
let aborted = false;
|
||||
const client = {
|
||||
async create() {
|
||||
return "upload";
|
||||
},
|
||||
async uploadPart(_id: string, _key: string, part: number) {
|
||||
uploaded.push(part);
|
||||
return `etag-${part}`;
|
||||
},
|
||||
async complete() {
|
||||
completed = true;
|
||||
},
|
||||
async abort() {
|
||||
aborted = true;
|
||||
},
|
||||
};
|
||||
await multipartUpload(
|
||||
client,
|
||||
"video",
|
||||
new Uint8Array(11 * 1024 * 1024),
|
||||
{ contentType: "video/mp4" },
|
||||
{ partBytes: 5 * 1024 * 1024, concurrency: 2 },
|
||||
);
|
||||
expect(uploaded).toEqual([1, 2, 3]);
|
||||
expect(completed).toBeTrue();
|
||||
expect(aborted).toBeFalse();
|
||||
});
|
||||
|
||||
test("temporary cleanup and video transcoding are bounded and injectable", async () => {
|
||||
let now = 10;
|
||||
const deleted: string[] = [];
|
||||
const cleaner = createTemporaryObjectCleaner(
|
||||
{
|
||||
async put() {},
|
||||
async get() {
|
||||
return null;
|
||||
},
|
||||
async delete(key) {
|
||||
deleted.push(key);
|
||||
},
|
||||
publicUrl() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{ now: () => now },
|
||||
);
|
||||
cleaner.track("tmp/a", 5);
|
||||
expect(await cleaner.cleanup()).toBe(0);
|
||||
now = 15;
|
||||
expect(await cleaner.cleanup()).toBe(1);
|
||||
expect(deleted).toEqual(["tmp/a"]);
|
||||
let command: string[] = [];
|
||||
await ffmpegVideoTranscoder({
|
||||
spawn(args) {
|
||||
command = args;
|
||||
return { exited: Promise.resolve(0) };
|
||||
},
|
||||
})("input.mov", "output.mp4", { format: "mp4", width: 1280, videoBitrateKbps: 2000 });
|
||||
expect(command).toContain("scale=1280:-2");
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { createResumableUploadManager, memoryResumableSessionStore } from "../src/index.ts";
|
||||
import type { StorageDriver } from "../src/index.ts";
|
||||
|
||||
function driver() {
|
||||
const objects = new Map<string, Uint8Array>();
|
||||
const storage: StorageDriver = {
|
||||
async put(key, data) {
|
||||
objects.set(key, data.slice());
|
||||
},
|
||||
async get(key) {
|
||||
const body = objects.get(key);
|
||||
return body ? { body, contentType: "application/octet-stream", size: body.length } : null;
|
||||
},
|
||||
async delete(key) {
|
||||
objects.delete(key);
|
||||
},
|
||||
publicUrl: () => null,
|
||||
};
|
||||
return { storage, objects };
|
||||
}
|
||||
|
||||
test("resumable uploads accept out-of-order idempotent chunks and assemble once", async () => {
|
||||
const { storage, objects } = driver();
|
||||
const manager = createResumableUploadManager({
|
||||
driver: storage,
|
||||
chunkSize: 3,
|
||||
maxBytes: 20,
|
||||
accept: ["text/plain"],
|
||||
});
|
||||
const session = await manager.create({ name: "hello.txt", type: "text/plain", size: 8 });
|
||||
expect(session.totalChunks).toBe(3);
|
||||
expect((await manager.uploadChunk(session.id, 1, new TextEncoder().encode("lo "))).complete).toBe(
|
||||
false,
|
||||
);
|
||||
await manager.uploadChunk(session.id, 0, new TextEncoder().encode("hel"));
|
||||
await manager.uploadChunk(session.id, 0, new TextEncoder().encode("hel"));
|
||||
const result = await manager.uploadChunk(session.id, 2, new TextEncoder().encode("!!"));
|
||||
expect(result.complete).toBe(true);
|
||||
expect(new TextDecoder().decode(objects.get(result.file!.key))).toBe("hello !!");
|
||||
expect(await manager.status(session.id)).toBeNull();
|
||||
});
|
||||
|
||||
test("resumable uploads enforce checksums, conflicts, limits, expiry, and cancellation", async () => {
|
||||
let now = 0;
|
||||
const { storage } = driver();
|
||||
const sessions = memoryResumableSessionStore();
|
||||
const manager = createResumableUploadManager({
|
||||
driver: storage,
|
||||
sessions,
|
||||
chunkSize: 2,
|
||||
maxBytes: 4,
|
||||
maxSessions: 1,
|
||||
ttlMs: 10,
|
||||
now: () => now,
|
||||
});
|
||||
const session = await manager.create({ name: "data.bin", size: 4 });
|
||||
await expect(manager.create({ name: "other.bin", size: 1 })).rejects.toThrow("too many");
|
||||
await expect(manager.uploadChunk(session.id, 0, new Uint8Array([1, 2]), "bad")).rejects.toThrow(
|
||||
"checksum",
|
||||
);
|
||||
await manager.uploadChunk(session.id, 0, new Uint8Array([1, 2]));
|
||||
await expect(manager.uploadChunk(session.id, 0, new Uint8Array([2, 1]))).rejects.toThrow(
|
||||
"different content",
|
||||
);
|
||||
expect(await manager.cancel(session.id)).toBe(true);
|
||||
const expiring = await manager.create({ name: "expire.bin", size: 2 });
|
||||
now = 11;
|
||||
expect(await manager.prune()).toBe(1);
|
||||
await expect(manager.uploadChunk(expiring.id, 0, new Uint8Array([1, 2]))).rejects.toThrow(
|
||||
"expired",
|
||||
);
|
||||
});
|
||||
@@ -61,3 +61,30 @@ test("signed file tokens reject tampering and expired payloads", async () => {
|
||||
expect(await verifySignedFileToken(`${token}x`, secret, 1_000)).toBeNull();
|
||||
expect(await verifySignedFileToken(token, secret, 2_000)).toBeNull();
|
||||
});
|
||||
|
||||
test("upload scanning rejects unsafe bytes before storage", async () => {
|
||||
const form = new FormData();
|
||||
form.set("file", new File(["virus"], "bad.txt", { type: "text/plain" }));
|
||||
await expect(
|
||||
upload("public", new Request("http://test/upload", { method: "POST", body: form }), {
|
||||
scan: async () => ({ safe: false, scanner: "test-av", reason: "signature" }),
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 422 });
|
||||
expect(await getStore("public").driver.get("bad.txt")).toBeNull();
|
||||
});
|
||||
|
||||
test("post-storage processor failure rolls back the object", async () => {
|
||||
const form = new FormData();
|
||||
form.set("file", new File(["image"], "photo.png", { type: "image/png" }));
|
||||
let key = "";
|
||||
await expect(
|
||||
upload("public", new Request("http://test/upload", { method: "POST", body: form }), {
|
||||
afterStore(file) {
|
||||
key = file.key;
|
||||
throw new Error("transform failed");
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 422 });
|
||||
expect(key).not.toBe("");
|
||||
expect(await getStore("public").driver.get(key)).toBeNull();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user