WorkRoot/WRNexusJS
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(syntax): make api-section scanning string/comment-aware

Browse Source 
Reuse tokenizer.readBalancedBraces string/comment skipping (extracted as
skipLiteralOrComment) for both section detection and slicing, instead of a
second hand-rolled brace counter. Fixes truncation on braces inside strings
and false-positive sectioned detection from keywords inside comments/strings.
Also switch api-sections.ts errors from plain Error to LexError so parser.ts
upgrades them to ParseError with an offset.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Clintchiz
2026-08-19 15:30:51 +05:30
co-authored by Claude Opus 5
parent 028c2a6d64
commit 323f57b32b
3 changed files with 203 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/syntax/src/tokenizer.ts
+47 -27
View File
@@ -31,8 +31,47 @@ export interface Token {
export class LexError extends Error {}
const isWs = (c: string) => c === " " || c === "\t" || c === "\n" || c === "\r";
const isIdentStart = (c: string) => /[A-Za-z_]/.test(c);
const isIdentPart = (c: string) => /[A-Za-z0-9_]/.test(c);
export const isIdentStart = (c: string) => /[A-Za-z_]/.test(c);
export const isIdentPart = (c: string) => /[A-Za-z0-9_]/.test(c);
/**
* Skip over a string/template literal or comment starting at `src[i]`, using
* the exact rules `readBalancedBraces` needs to stay comment- and
* string-aware: `/* block *\/` comments anywhere, `//` line comments only at
* the start of a line (so a bare `https://…` in view text isn't mistaken for
* one), and `"`, `'`, `` ` `` strings with backslash escapes.
*
* Returns the index just past what it skipped, or `null` when `src[i]` isn't
* the start of one of those. Exported so any other raw-body scanner that
* needs to walk `.wrn` source without tripping over strings or comments
* (e.g. the `api` section scanner) shares this logic instead of
* reimplementing it — a second hand-rolled scanner is how apostrophes in
* prose used to swallow braces.
*/
export function skipLiteralOrComment(src: string, i: number, atLineStart: boolean): number | null {
const c = src[i];
if (c === "/" && src[i + 1] === "*") {
const close = src.indexOf("*/", i + 2);
return close === -1 ? src.length : close + 2;
}
if (atLineStart && c === "/" && src[i + 1] === "/") {
const newline = src.indexOf("\n", i + 2);
return newline === -1 ? src.length : newline;
}
if (c === '"' || c === "'" || c === "`") {
let j = i + 1;
while (j < src.length) {
if (src[j] === "\\") {
j += 2;
continue;
}
if (src[j] === c) return j + 1;
j++;
}
return src.length;
}
return null;
}
export class Lexer {
pos = 0;
@@ -312,46 +351,26 @@ export class Lexer {
const start = this.pos + 1;
let depth = 0;
let i = this.pos;
let str: string | null = null;
/** True while only whitespace has been seen since the last newline. */
let atLineStart = false;
for (; i < src.length; i++) {
while (i < src.length) {
const c = src[i]!;
if (str) {
if (c === "\\") {
i++;
continue;
}
if (c === str) str = null;
continue;
}
if (c === "\n") {
atLineStart = true;
i++;
continue;
}
if (c === "/" && src[i + 1] === "*") {
const close = src.indexOf("*/", i + 2);
if (close === -1) break; // unterminated: fall through to the error
i = close + 1;
const skipped = skipLiteralOrComment(src, i, atLineStart);
if (skipped !== null) {
i = skipped;
atLineStart = false;
continue;
}
if (atLineStart && c === "/" && src[i + 1] === "/") {
const newline = src.indexOf("\n", i + 2);
if (newline === -1) break;
i = newline - 1; // let the loop's own increment land on the newline
continue;
}
if (c !== " " && c !== "\t" && c !== "\r") atLineStart = false;
if (c === '"' || c === "'" || c === "`") {
str = c;
continue;
}
if (c === "{") depth++;
else if (c === "}") {
depth--;
@@ -360,6 +379,7 @@ export class Lexer {
return src.slice(start, i);
}
}
i++;
}
throw new LexError(`Unbalanced braces starting at offset ${this.pos}`);
}