export interface BenchmarkOptions { iterations?: number; warmup?: number; clock?: () => number; setup?: () => void | Promise; teardown?: () => void | Promise; } export interface BenchmarkResult { name: string; iterations: number; totalMs: number; meanMs: number; minMs: number; maxMs: number; p50Ms: number; p95Ms: number; p99Ms: number; operationsPerSecond: number; samples: number[]; } export interface RegressionBudget { meanPercent?: number; p95Percent?: number; maxAbsoluteMs?: number; minOperationsPerSecond?: number; } export interface RegressionViolation { metric: "meanMs" | "p95Ms" | "maxMs" | "operationsPerSecond"; baseline?: number; current: number; limit: number; message: string; } export function percentile(values: readonly number[], quantile: number): number { if (!values.length) return 0; const sorted = [...values].sort((a, b) => a - b); const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(quantile * sorted.length) - 1)); return sorted[index]!; } export async function runBenchmark( name: string, operation: () => void | Promise, options: BenchmarkOptions = {}, ): Promise { const iterations = Math.max(1, Math.floor(options.iterations ?? 100)); const warmup = Math.max(0, Math.floor(options.warmup ?? Math.min(10, iterations))); const clock = options.clock ?? (() => performance.now()); await options.setup?.(); try { for (let index = 0; index < warmup; index++) await operation(); const samples: number[] = []; for (let index = 0; index < iterations; index++) { const start = clock(); await operation(); samples.push(Math.max(0, clock() - start)); } const totalMs = samples.reduce((sum, sample) => sum + sample, 0); const meanMs = totalMs / samples.length; return { name, iterations, totalMs, meanMs, minMs: Math.min(...samples), maxMs: Math.max(...samples), p50Ms: percentile(samples, 0.5), p95Ms: percentile(samples, 0.95), p99Ms: percentile(samples, 0.99), operationsPerSecond: meanMs === 0 ? Number.POSITIVE_INFINITY : 1000 / meanMs, samples, }; } finally { await options.teardown?.(); } } export function compareBenchmark( current: BenchmarkResult, baseline: BenchmarkResult | undefined, budget: RegressionBudget = {}, ): RegressionViolation[] { const violations: RegressionViolation[] = []; if (baseline && budget.meanPercent !== undefined) { const limit = baseline.meanMs * (1 + budget.meanPercent / 100); if (current.meanMs > limit) { violations.push({ metric: "meanMs", baseline: baseline.meanMs, current: current.meanMs, limit, message: `Mean latency regressed by more than ${budget.meanPercent}%.`, }); } } if (baseline && budget.p95Percent !== undefined) { const limit = baseline.p95Ms * (1 + budget.p95Percent / 100); if (current.p95Ms > limit) { violations.push({ metric: "p95Ms", baseline: baseline.p95Ms, current: current.p95Ms, limit, message: `P95 latency regressed by more than ${budget.p95Percent}%.`, }); } } if (budget.maxAbsoluteMs !== undefined && current.maxMs > budget.maxAbsoluteMs) { violations.push({ metric: "maxMs", current: current.maxMs, limit: budget.maxAbsoluteMs, message: `Maximum latency exceeds ${budget.maxAbsoluteMs} ms.`, }); } if ( budget.minOperationsPerSecond !== undefined && current.operationsPerSecond < budget.minOperationsPerSecond ) { violations.push({ metric: "operationsPerSecond", current: current.operationsPerSecond, limit: budget.minOperationsPerSecond, message: `Throughput is below ${budget.minOperationsPerSecond} operations/second.`, }); } return violations; } export function assertBenchmarkBudget( current: BenchmarkResult, baseline: BenchmarkResult | undefined, budget: RegressionBudget, ): void { const violations = compareBenchmark(current, baseline, budget); if (violations.length) { throw new Error( `Benchmark '${current.name}' failed:\n${violations.map((item) => `- ${item.message}`).join("\n")}`, ); } }