457 lines
15 KiB
TypeScript
457 lines
15 KiB
TypeScript
import { expect, test } from "bun:test";
|
|
import { CORE_DESIGN_SYSTEMS } from "@wrnexus/design-system";
|
|
import { designRevision, DEFAULT_DESIGN_BRIEF } from "@wrnexus/design-studio";
|
|
import {
|
|
AI_TIMEOUT_MS,
|
|
AI_OUTPUT_TOKENS,
|
|
apimartProvider,
|
|
studioAI,
|
|
} from "../app/lib/studio-ai.ts";
|
|
import { addStudioGuide } from "../app/lib/studio-guide.ts";
|
|
|
|
const proposals = ["2px", "4px", "8px"].map((value, index) => ({
|
|
name: `Direction ${index + 1}`,
|
|
rationale: "A different control shape.",
|
|
patch: [{ op: "set", path: "/tokens/shape.control/value", value }],
|
|
}));
|
|
const completion = () =>
|
|
Response.json({
|
|
choices: [
|
|
{
|
|
finish_reason: "stop",
|
|
message: { content: JSON.stringify({ alternatives: proposals }) },
|
|
},
|
|
],
|
|
});
|
|
const mockFetch = (
|
|
fn: (input: string | URL | Request, init?: RequestInit) => Promise<Response>,
|
|
) => fn as typeof fetch;
|
|
const system = CORE_DESIGN_SYSTEMS.square;
|
|
const input = async (locks: string[] = []) => ({
|
|
system,
|
|
locks,
|
|
brief: {
|
|
...DEFAULT_DESIGN_BRIEF,
|
|
product: "A website",
|
|
audience: "Customers",
|
|
},
|
|
kind: "directions",
|
|
baseRevision: await designRevision(system, locks),
|
|
});
|
|
const request = async (body?: unknown, origin = "https://components.example") =>
|
|
new Request("https://components.example/design-studio/api/generate", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json", origin },
|
|
body: JSON.stringify(body ?? (await input())),
|
|
});
|
|
|
|
test("APIMart uses the specified model, bearer authentication, non-streaming JSON and cancellation", async () => {
|
|
const controller = new AbortController();
|
|
const provider = apimartProvider(
|
|
"test-secret",
|
|
mockFetch(async (url, init) => {
|
|
expect(String(url)).toBe("https://api.apimart.ai/v1/chat/completions");
|
|
expect(new Headers(init?.headers).get("authorization")).toBe(
|
|
"Bearer test-secret",
|
|
);
|
|
expect(init?.signal).toBe(controller.signal);
|
|
const body = JSON.parse(init?.body as string);
|
|
expect(body.model).toBe("gpt-5-mini");
|
|
expect(body.stream).toBe(false);
|
|
expect(body.max_tokens).toBe(AI_OUTPUT_TOKENS);
|
|
expect(body.messages).toEqual([
|
|
{ role: "system", content: "rules" },
|
|
{ role: "user", content: "brief" },
|
|
]);
|
|
return completion();
|
|
}),
|
|
);
|
|
const result = await provider.generate("brief", {
|
|
system: "rules",
|
|
signal: controller.signal,
|
|
});
|
|
expect(JSON.parse(result.value).alternatives).toHaveLength(3);
|
|
});
|
|
|
|
test("AI proposals use engine validation, preserve draft source, locks and revision checks", async () => {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
calls++;
|
|
return completion();
|
|
}),
|
|
);
|
|
const original = JSON.stringify(system);
|
|
const response = await handler(await request());
|
|
const batch = await response!.json();
|
|
expect(batch.alternatives).toHaveLength(3);
|
|
expect(batch.provenance.provider).toBe("AI");
|
|
expect(batch.provenance.model).toBeUndefined();
|
|
expect(JSON.stringify(batch)).not.toMatch(/APIMart|gpt-5-mini|custom-model/i);
|
|
expect(JSON.stringify(system)).toBe(original);
|
|
const locked = await handler(
|
|
await request(await input(["/tokens/shape.control"])),
|
|
);
|
|
expect((await locked!.json()).alternatives).toHaveLength(0);
|
|
const before = calls;
|
|
expect(
|
|
(await handler(
|
|
await request({ ...(await input()), baseRevision: "a".repeat(64) }),
|
|
))!.status,
|
|
).toBe(409);
|
|
expect(calls).toBe(before);
|
|
});
|
|
|
|
test("metadata and errors never contain the key or raw upstream response", async () => {
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(
|
|
async () =>
|
|
new Response("test-secret upstream credentials", { status: 401 }),
|
|
),
|
|
);
|
|
const meta = await handler(
|
|
new Request("https://components.example/design-studio/api/generation"),
|
|
);
|
|
const text = await meta!.text();
|
|
expect(text).not.toMatch(/apimart|gpt-5-mini/i);
|
|
expect(JSON.parse(text).timeoutMs).toBe(AI_TIMEOUT_MS);
|
|
expect(AI_TIMEOUT_MS).toBe(60_000);
|
|
expect(text).not.toContain("test-secret");
|
|
const failure = await handler(await request());
|
|
const body = await failure!.text();
|
|
expect(body).not.toContain("test-secret");
|
|
expect(body).not.toContain("upstream credentials");
|
|
expect(body).not.toMatch(/APIMart|APIMART_API_KEY|gpt-5-mini/i);
|
|
expect(body).toContain("WRN-STUDIO-PROVIDER");
|
|
});
|
|
|
|
test("a completion beyond the old 20-second deadline produces three validated alternatives", async () => {
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async (_url, init) => {
|
|
await Bun.sleep(21_000);
|
|
expect(init?.signal?.aborted).toBe(false);
|
|
return completion();
|
|
}),
|
|
);
|
|
const original = JSON.stringify(system);
|
|
const response = await handler(await request());
|
|
expect(response!.status).toBe(200);
|
|
const batch = await response!.json();
|
|
expect(batch.alternatives).toHaveLength(3);
|
|
expect(batch.provenance.provider).toBe("AI");
|
|
expect(JSON.stringify(system)).toBe(original);
|
|
}, 30_000);
|
|
|
|
test("cancellation still aborts the upstream request and keeps the draft unchanged", async () => {
|
|
const controller = new AbortController();
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async (_url, init) => {
|
|
controller.abort();
|
|
expect(init?.signal?.aborted).toBe(true);
|
|
throw new DOMException("Cancelled", "AbortError");
|
|
}),
|
|
);
|
|
const original = JSON.stringify(system);
|
|
const base = await request();
|
|
const response = await handler(
|
|
new Request(base, { signal: controller.signal }),
|
|
);
|
|
expect(response!.status).toBe(408);
|
|
expect(await response!.text()).toContain("draft is unchanged");
|
|
expect(JSON.stringify(system)).toBe(original);
|
|
});
|
|
|
|
test("same-origin checks, body bounds and local fallback remain enforced", async () => {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
calls++;
|
|
return completion();
|
|
}),
|
|
);
|
|
expect(
|
|
(await handler(await request(await input(), "https://other.example")))!
|
|
.status,
|
|
).toBe(403);
|
|
expect(
|
|
(await handler(await request({ huge: "x".repeat(2_000_001) })))!.status,
|
|
).toBe(413);
|
|
expect(calls).toBe(0);
|
|
expect(await studioAI()(await request())).toBeUndefined();
|
|
expect(
|
|
addStudioGuide(
|
|
'<head><script type="module"></script></head><div id="diagnostics"></div>',
|
|
false,
|
|
),
|
|
).not.toMatch(/APIMART|gpt-/i);
|
|
expect(addStudioGuide('<div id="diagnostics"></div>', true)).toContain(
|
|
"AI design assistance is available",
|
|
);
|
|
});
|
|
|
|
test("a public AI endpoint enforces a bounded generation rate", async () => {
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => completion()),
|
|
);
|
|
for (let index = 0; index < 10; index++)
|
|
expect((await handler(await request()))!.status).toBe(200);
|
|
expect((await handler(await request()))!.status).toBe(429);
|
|
});
|
|
|
|
test("temporary APIMart failures retry once and return three validated designs", async () => {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async (_url, init) => {
|
|
calls++;
|
|
expect(JSON.parse(init!.body as string).max_tokens).toBe(8192);
|
|
return calls === 1
|
|
? new Response("sensitive upstream context test-secret", {
|
|
status: 503,
|
|
})
|
|
: completion();
|
|
}),
|
|
);
|
|
const batch = await (await handler(await request()))!.json();
|
|
expect(calls).toBe(2);
|
|
expect(batch.alternatives).toHaveLength(3);
|
|
expect(batch.provenance.attempts).toBe(2);
|
|
expect(batch.provenance.maxOutputTokens).toBe(8192);
|
|
});
|
|
|
|
test("authentication and quota failures provide actionable redacted diagnostics without retrying", async () => {
|
|
for (const [status, code, hint] of [
|
|
[401, "invalid_api_key", "server configuration issue"],
|
|
[402, "insufficient_quota", "service usage limit"],
|
|
[404, "model_not_found", "server configuration issue"],
|
|
] as const) {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
calls++;
|
|
return Response.json(
|
|
{ error: { code, message: "test-secret private brief" } },
|
|
{ status },
|
|
);
|
|
}),
|
|
);
|
|
const body = await (await handler(await request()))!.text();
|
|
expect(calls).toBe(1);
|
|
expect(body).toContain(hint);
|
|
expect(body).not.toContain("test-secret");
|
|
expect(body).not.toContain("private brief");
|
|
}
|
|
});
|
|
|
|
test("truncated and empty completions retry but never exceed two total upstream calls", async () => {
|
|
for (const kind of ["length", "empty", "invalid-then-outage"]) {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
calls++;
|
|
if (kind === "invalid-then-outage")
|
|
return calls === 1
|
|
? Response.json({
|
|
choices: [{ message: { content: "invalid JSON" } }],
|
|
})
|
|
: new Response("private upstream body", { status: 502 });
|
|
return calls === 1
|
|
? Response.json({
|
|
choices: [
|
|
{
|
|
finish_reason: kind === "length" ? "length" : "stop",
|
|
message: { content: "" },
|
|
},
|
|
],
|
|
})
|
|
: completion();
|
|
}),
|
|
);
|
|
const batch = await (await handler(await request()))!.json();
|
|
expect(calls).toBe(2);
|
|
expect(batch.provenance.attempts).toBe(2);
|
|
expect(batch.alternatives).toHaveLength(
|
|
kind === "invalid-then-outage" ? 0 : 3,
|
|
);
|
|
}
|
|
});
|
|
|
|
test("network and repeated rate failures remain bounded and describe the actual failure category", async () => {
|
|
for (const network of [true, false]) {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
calls++;
|
|
if (network) throw Error("private test-secret TLS details");
|
|
return new Response("private test-secret", { status: 429 });
|
|
}),
|
|
);
|
|
const body = await (await handler(await request()))!.text();
|
|
expect(calls).toBe(2);
|
|
expect(body).toContain(
|
|
network ? "could not reach the AI service" : "rate limiting",
|
|
);
|
|
expect(body).not.toContain("test-secret");
|
|
}
|
|
});
|
|
|
|
test("a retry followed by an invalid design does not spend a third request or mislabel a schema error", async () => {
|
|
let calls = 0;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
calls++;
|
|
return calls === 1
|
|
? new Response("unavailable", { status: 503 })
|
|
: Response.json({ choices: [{ message: { content: "not JSON" } }] });
|
|
}),
|
|
);
|
|
const batch = await (await handler(await request()))!.json();
|
|
expect(calls).toBe(2);
|
|
expect(batch.alternatives).toHaveLength(0);
|
|
expect(batch.diagnostics[0].code).toBe("WRN-STUDIO-PROPOSAL");
|
|
expect(batch.diagnostics[0].message).toContain("schema");
|
|
});
|
|
|
|
test("async generation acknowledges immediately and polling retrieves the validated batch", async () => {
|
|
let finish!: () => void;
|
|
let entered!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
entered = resolve;
|
|
});
|
|
const blocked = new Promise<void>((resolve) => {
|
|
finish = resolve;
|
|
});
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async () => {
|
|
entered();
|
|
await blocked;
|
|
return completion();
|
|
}),
|
|
);
|
|
const original = await request();
|
|
const headers = new Headers(original.headers);
|
|
headers.set("prefer", "respond-async");
|
|
const accepted = await handler(new Request(original, { headers }));
|
|
expect(accepted!.status).toBe(202);
|
|
const { job } = await accepted!.json();
|
|
await started;
|
|
const url = `https://components.example/design-studio/api/generate/jobs/${job.id}`;
|
|
expect((await (await handler(new Request(url)))!.json()).status).toBe(
|
|
"pending",
|
|
);
|
|
expect(
|
|
(await handler(
|
|
new Request(url, { headers: { origin: "https://other.example" } }),
|
|
))!.status,
|
|
).toBe(403);
|
|
finish();
|
|
let result: any;
|
|
for (let i = 0; i < 50; i++) {
|
|
result = await (await handler(new Request(url)))!.json();
|
|
if (result.status !== "pending") break;
|
|
await Bun.sleep(10);
|
|
}
|
|
expect(result.status).toBe("complete");
|
|
expect(result.result.alternatives).toHaveLength(3);
|
|
expect(
|
|
(await handler(
|
|
new Request(url.replace(job.id, "00000000-0000-4000-8000-000000000000")),
|
|
))!.status,
|
|
).toBe(410);
|
|
});
|
|
|
|
test("background generation cancellation aborts the provider and leaves no proposals", async () => {
|
|
let entered!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
entered = resolve;
|
|
});
|
|
let aborted = false;
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async (_url, init) => {
|
|
entered();
|
|
return await new Promise<Response>((_, reject) => {
|
|
init!.signal!.addEventListener(
|
|
"abort",
|
|
() => {
|
|
aborted = true;
|
|
reject(new DOMException("cancelled", "AbortError"));
|
|
},
|
|
{ once: true },
|
|
);
|
|
});
|
|
}),
|
|
);
|
|
const original = await request();
|
|
const headers = new Headers(original.headers);
|
|
headers.set("prefer", "respond-async");
|
|
const { job } = await (await handler(
|
|
new Request(original, { headers }),
|
|
))!.json();
|
|
await started;
|
|
const response = await handler(
|
|
new Request(
|
|
`https://components.example/design-studio/api/generate/jobs/${job.id}`,
|
|
{ method: "DELETE" },
|
|
),
|
|
);
|
|
expect((await response!.json()).status).toBe("cancelled");
|
|
expect(aborted).toBe(true);
|
|
});
|
|
|
|
test("the configured model is used consistently for metadata and upstream requests", async () => {
|
|
let selected = "";
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(async (_url, init) => {
|
|
selected = JSON.parse(init!.body as string).model;
|
|
return completion();
|
|
}),
|
|
"custom-model",
|
|
);
|
|
const metadata = await (await handler(
|
|
new Request("https://components.example/design-studio/api/generation"),
|
|
))!.json();
|
|
expect(metadata.model).toBeUndefined();
|
|
const batch = await (await handler(await request()))!.json();
|
|
expect(selected).toBe("custom-model");
|
|
expect(batch.provenance.model).toBeUndefined();
|
|
expect(JSON.stringify(batch)).not.toMatch(/APIMart|gpt-5-mini|custom-model/i);
|
|
expect(addStudioGuide('<div id="diagnostics"></div>', true)).not.toMatch(
|
|
/APIMart|custom-model|gpt-/i,
|
|
);
|
|
});
|
|
|
|
test("all public AI failure categories hide provider and model configuration", async () => {
|
|
for (const status of [400, 401, 402, 403, 404, 429, 500]) {
|
|
const handler = studioAI(
|
|
"test-secret",
|
|
mockFetch(
|
|
async () =>
|
|
new Response(
|
|
JSON.stringify({
|
|
error: { message: "APIMart custom-secret-model" },
|
|
}),
|
|
{ status },
|
|
),
|
|
),
|
|
"custom-secret-model",
|
|
);
|
|
const response = await handler(await request());
|
|
expect(await response!.text()).not.toMatch(
|
|
/APIMart|custom-secret-model|test-secret/i,
|
|
);
|
|
}
|
|
});
|