From 323f57b32bcc87ba13507ea34f7111fa26dd9282 Mon Sep 17 00:00:00 2001 From: Ajay Ghanwat Date: Wed, 19 Aug 2026 15:30:51 +0530 Subject: [PATCH] fix(syntax): make api-section scanning string/comment-aware Reuse tokenizer.readBalancedBraces string/comment skipping (extracted as skipLiteralOrComment) for both section detection and slicing, instead of a second hand-rolled brace counter. Fixes truncation on braces inside strings and false-positive sectioned detection from keywords inside comments/strings. Also switch api-sections.ts errors from plain Error to LexError so parser.ts upgrades them to ParseError with an offset. Co-Authored-By: Claude Opus 5 --- packages/syntax/src/api-sections.ts | 140 +++++++++++++++++++++---- packages/syntax/src/tokenizer.ts | 74 ++++++++----- packages/syntax/test/api-block.test.ts | 39 +++++++ 3 files changed, 203 insertions(+), 50 deletions(-) diff --git a/packages/syntax/src/api-sections.ts b/packages/syntax/src/api-sections.ts index bfa76b5f..67b02928 100644 --- a/packages/syntax/src/api-sections.ts +++ b/packages/syntax/src/api-sections.ts @@ -4,7 +4,14 @@ * Returns null when no section keyword is present, which is how the legacy * bare-body form stays valid: the caller keeps treating the body as the * response expression. + * + * Detection and slicing both drive the tokenizer's own string/comment-aware + * scanning (`skipLiteralOrComment`, `Lexer.readBalancedBraces`) instead of a + * second hand-rolled brace counter, so a `}` inside a string or a `request {` + * mentioned in a comment can't be mistaken for a real section. */ +import { Lexer, LexError, isIdentPart, isIdentStart, skipLiteralOrComment } from "./tokenizer.ts"; + export interface ApiFieldDecl { name: string; optional: boolean; @@ -19,38 +26,122 @@ export interface ApiSections { } const SECTION_NAMES = ["request", "response", "error"] as const; +const REQUEST_SUBSECTION_NAMES = ["parameters", "body"] as const; -/** Slice the balanced `{ ... }` that follows `keyword`, or null when absent. */ -function sectionBody(source: string, keyword: string): string | null { - const match = new RegExp(`(^|[^A-Za-z0-9_$])${keyword}\\s*\\{`).exec(source); - if (!match) return null; +interface Span { + text: string; + /** Offset of `text[0]` within the source that was scanned. */ + start: number; +} - const open = source.indexOf("{", match.index + match[1]!.length); +/** + * Walk `source` at brace-depth 0, looking for `name { ... }` where `name` is + * one of `names`. Strings, template literals, and comments are skipped via + * `skipLiteralOrComment` — the same rules `readBalancedBraces` uses — so a + * keyword mentioned inside a string or comment, or nested inside an unrelated + * `{ }` (e.g. an object literal in a legacy body), is never mistaken for a + * section. Matched blocks are sliced via `Lexer.readBalancedBraces()` itself, + * not a reimplementation of it. + */ +function scanTopLevelBlocks(source: string, names: readonly string[]): Map { + const found = new Map(); + const lx = new Lexer(source); let depth = 0; + let i = 0; + let atLineStart = true; - for (let index = open; index < source.length; index++) { - const character = source[index]; - if (character === "{") depth++; - else if (character === "}") { - depth--; - if (depth === 0) return source.slice(open + 1, index); + while (i < source.length) { + const c = source[i]!; + + if (c === "\n") { + atLineStart = true; + i++; + continue; } + + const skipped = skipLiteralOrComment(source, i, atLineStart); + if (skipped !== null) { + i = skipped; + atLineStart = false; + continue; + } + + if (c !== " " && c !== "\t" && c !== "\r") atLineStart = false; + + if (depth === 0 && isIdentStart(c)) { + let j = i + 1; + while (j < source.length && isIdentPart(source[j]!)) j++; + const word = source.slice(i, j); + + // Skip trivia between the identifier and a possible '{' without + // treating anything in between as significant yet. + let k = j; + let lineStartAtK = false; + while (k < source.length) { + const kc = source[k]!; + if (kc === " " || kc === "\t" || kc === "\r") { + k++; + continue; + } + if (kc === "\n") { + lineStartAtK = true; + k++; + continue; + } + const kSkipped = skipLiteralOrComment(source, k, lineStartAtK); + if (kSkipped !== null) { + k = kSkipped; + lineStartAtK = false; + continue; + } + break; + } + + if (names.includes(word) && source[k] === "{") { + lx.pos = k; + const start = k + 1; + const text = lx.readBalancedBraces(); + if (!found.has(word)) found.set(word, { text, start }); + i = lx.pos; + continue; + } + + i = j; + continue; + } + + if (c === "{") depth++; + else if (c === "}") depth = Math.max(0, depth - 1); + i++; } - throw new Error(`Unclosed "${keyword}" section in an api block`); + return found; +} + +/** Rebase a span captured from `outer.text` back onto the original source. */ +function absolutize(span: Span | undefined, outer: Span | undefined): Span | undefined { + if (!span) return undefined; + return outer ? { text: span.text, start: outer.start + span.start } : span; } /** `name?: string` -> { name, optional, type }. Blank lines and comments are skipped. */ -function parseFields(source: string): ApiFieldDecl[] { +function parseFields(span: Span | undefined): ApiFieldDecl[] { + if (!span) return []; const fields: ApiFieldDecl[] = []; + let cursor = 0; + + for (const rawLine of span.text.split("\n")) { + const lineOffset = span.start + cursor; + cursor += rawLine.length + 1; - for (const rawLine of source.split("\n")) { const line = rawLine.trim().replace(/,$/, ""); if (!line || line.startsWith("//")) continue; const match = /^([A-Za-z_$][A-Za-z0-9_$]*)(\?)?\s*:\s*(.+)$/.exec(line); if (!match) { - throw new Error(`Expected "name: type" in an api request section, got "${line}"`); + throw new LexError( + `Expected "name: type" in an api request section, got "${line}" at offset ${lineOffset}`, + ); } fields.push({ name: match[1]!, optional: match[2] === "?", type: match[3]!.trim() }); @@ -60,20 +151,23 @@ function parseFields(source: string): ApiFieldDecl[] { } export function parseApiSections(source: string): ApiSections | null { - const present = SECTION_NAMES.some((name) => sectionBody(source, name) !== null); - if (!present) return null; + const top = scanTopLevelBlocks(source, SECTION_NAMES); + if (top.size === 0) return null; - const request = sectionBody(source, "request"); + const request = top.get("request"); + const sub = request + ? scanTopLevelBlocks(request.text, REQUEST_SUBSECTION_NAMES) + : new Map(); return { - parameters: request ? parseFields(sectionBody(request, "parameters") ?? "") : [], - body: request ? parseFields(sectionBody(request, "body") ?? "") : [], - response: sectionBody(source, "response") ?? "", - error: sectionBody(source, "error") ?? "", + parameters: parseFields(absolutize(sub.get("parameters"), request)), + body: parseFields(absolutize(sub.get("body"), request)), + response: top.get("response")?.text ?? "", + error: top.get("error")?.text ?? "", }; } /** True when the block declares a `request` section. */ export function hasRequestSection(source: string): boolean { - return sectionBody(source, "request") !== null; + return scanTopLevelBlocks(source, SECTION_NAMES).has("request"); } diff --git a/packages/syntax/src/tokenizer.ts b/packages/syntax/src/tokenizer.ts index 2ee3e713..a8e2dd76 100644 --- a/packages/syntax/src/tokenizer.ts +++ b/packages/syntax/src/tokenizer.ts @@ -31,8 +31,47 @@ export interface Token { export class LexError extends Error {} const isWs = (c: string) => c === " " || c === "\t" || c === "\n" || c === "\r"; -const isIdentStart = (c: string) => /[A-Za-z_]/.test(c); -const isIdentPart = (c: string) => /[A-Za-z0-9_]/.test(c); +export const isIdentStart = (c: string) => /[A-Za-z_]/.test(c); +export const isIdentPart = (c: string) => /[A-Za-z0-9_]/.test(c); + +/** + * Skip over a string/template literal or comment starting at `src[i]`, using + * the exact rules `readBalancedBraces` needs to stay comment- and + * string-aware: `/* block *\/` comments anywhere, `//` line comments only at + * the start of a line (so a bare `https://…` in view text isn't mistaken for + * one), and `"`, `'`, `` ` `` strings with backslash escapes. + * + * Returns the index just past what it skipped, or `null` when `src[i]` isn't + * the start of one of those. Exported so any other raw-body scanner that + * needs to walk `.wrn` source without tripping over strings or comments + * (e.g. the `api` section scanner) shares this logic instead of + * reimplementing it — a second hand-rolled scanner is how apostrophes in + * prose used to swallow braces. + */ +export function skipLiteralOrComment(src: string, i: number, atLineStart: boolean): number | null { + const c = src[i]; + if (c === "/" && src[i + 1] === "*") { + const close = src.indexOf("*/", i + 2); + return close === -1 ? src.length : close + 2; + } + if (atLineStart && c === "/" && src[i + 1] === "/") { + const newline = src.indexOf("\n", i + 2); + return newline === -1 ? src.length : newline; + } + if (c === '"' || c === "'" || c === "`") { + let j = i + 1; + while (j < src.length) { + if (src[j] === "\\") { + j += 2; + continue; + } + if (src[j] === c) return j + 1; + j++; + } + return src.length; + } + return null; +} export class Lexer { pos = 0; @@ -312,46 +351,26 @@ export class Lexer { const start = this.pos + 1; let depth = 0; let i = this.pos; - let str: string | null = null; /** True while only whitespace has been seen since the last newline. */ let atLineStart = false; - for (; i < src.length; i++) { + while (i < src.length) { const c = src[i]!; - if (str) { - if (c === "\\") { - i++; - continue; - } - if (c === str) str = null; - continue; - } if (c === "\n") { atLineStart = true; + i++; continue; } - if (c === "/" && src[i + 1] === "*") { - const close = src.indexOf("*/", i + 2); - if (close === -1) break; // unterminated: fall through to the error - i = close + 1; + const skipped = skipLiteralOrComment(src, i, atLineStart); + if (skipped !== null) { + i = skipped; atLineStart = false; continue; } - if (atLineStart && c === "/" && src[i + 1] === "/") { - const newline = src.indexOf("\n", i + 2); - if (newline === -1) break; - i = newline - 1; // let the loop's own increment land on the newline - continue; - } - if (c !== " " && c !== "\t" && c !== "\r") atLineStart = false; - if (c === '"' || c === "'" || c === "`") { - str = c; - continue; - } if (c === "{") depth++; else if (c === "}") { depth--; @@ -360,6 +379,7 @@ export class Lexer { return src.slice(start, i); } } + i++; } throw new LexError(`Unbalanced braces starting at offset ${this.pos}`); } diff --git a/packages/syntax/test/api-block.test.ts b/packages/syntax/test/api-block.test.ts index 83bd6b87..fef2eb5f 100644 --- a/packages/syntax/test/api-block.test.ts +++ b/packages/syntax/test/api-block.test.ts @@ -96,3 +96,42 @@ test("request inside an ssr block is rejected with a message naming the restrict expect(() => parse(source)).toThrow(/ssr[\s\S]*request/i); }); + +test("a brace inside a string literal in the response body does not truncate the section", () => { + const ast = parse( + page(` api searchUsers POST /api/users { + request { + body { + name?: string + } + } + + response { + return "a } weird string" + } + + error { + return [] + } + }`), + ); + + const block = ast.dataApis[0]!; + expect(block.sections?.response.trim()).toBe('return "a } weird string"'); + expect(block.sections?.error.trim()).toBe("return []"); +}); + +test("a legacy block whose comment or string mentions a section keyword stays legacy", () => { + const ast = parse( + page(` api legacyUsers GET /api/users { + // fall back to a manual request { } if this fails + return "response { not a section }" + }`), + ); + + const block = ast.dataApis[0]!; + expect(block.sections).toBeUndefined(); + expect(block.body.trim()).toBe( + '// fall back to a manual request { } if this fails\n return "response { not a section }"', + ); +});