332 lines
9.2 KiB
TypeScript
332 lines
9.2 KiB
TypeScript
/**
|
|
* Lexer for the `.wrn` language.
|
|
*
|
|
* `.wrn` mixes a small structural grammar (page/state/view/api/realtime) with
|
|
* raw JavaScript bodies. A pure token stream can't represent the raw JS, so the
|
|
* lexer is driven on demand by the parser: it yields structural tokens via
|
|
* `next()`/`peek()`, and exposes `readBalancedBraces()`, `readPath()` and
|
|
* `readToLineEnd()` for the parser to grab raw spans when grammar demands it.
|
|
*/
|
|
|
|
export type TokenType =
|
|
| "ident"
|
|
| "string"
|
|
| "lbrace"
|
|
| "rbrace"
|
|
| "lparen"
|
|
| "rparen"
|
|
| "at"
|
|
| "eq"
|
|
| "colon"
|
|
| "comma"
|
|
| "question"
|
|
| "eof";
|
|
|
|
export interface Token {
|
|
type: TokenType;
|
|
value: string;
|
|
pos: number;
|
|
}
|
|
|
|
export class LexError extends Error {}
|
|
|
|
const isWs = (c: string) => c === " " || c === "\t" || c === "\n" || c === "\r";
|
|
const isIdentStart = (c: string) => /[A-Za-z_]/.test(c);
|
|
const isIdentPart = (c: string) => /[A-Za-z0-9_]/.test(c);
|
|
|
|
export class Lexer {
|
|
pos = 0;
|
|
constructor(public readonly src: string) {}
|
|
|
|
/** Skip whitespace and `// line comments`. */
|
|
private skipTrivia(): void {
|
|
const { src } = this;
|
|
while (this.pos < src.length) {
|
|
const c = src[this.pos]!;
|
|
if (isWs(c)) {
|
|
this.pos++;
|
|
continue;
|
|
}
|
|
if (c === "/" && src[this.pos + 1] === "/") {
|
|
while (this.pos < src.length && src[this.pos] !== "\n") this.pos++;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
/** Read and consume the next structural token. */
|
|
next(): Token {
|
|
this.skipTrivia();
|
|
const { src } = this;
|
|
const pos = this.pos;
|
|
if (pos >= src.length) return { type: "eof", value: "", pos };
|
|
|
|
const c = src[pos]!;
|
|
switch (c) {
|
|
case "{":
|
|
this.pos++;
|
|
return { type: "lbrace", value: c, pos };
|
|
case "}":
|
|
this.pos++;
|
|
return { type: "rbrace", value: c, pos };
|
|
case "(":
|
|
this.pos++;
|
|
return { type: "lparen", value: c, pos };
|
|
case ")":
|
|
this.pos++;
|
|
return { type: "rparen", value: c, pos };
|
|
case "@":
|
|
this.pos++;
|
|
return { type: "at", value: c, pos };
|
|
case "=":
|
|
this.pos++;
|
|
return { type: "eq", value: c, pos };
|
|
case ":":
|
|
this.pos++;
|
|
return { type: "colon", value: c, pos };
|
|
case ",":
|
|
this.pos++;
|
|
return { type: "comma", value: c, pos };
|
|
case "?":
|
|
this.pos++;
|
|
return { type: "question", value: c, pos };
|
|
case '"':
|
|
case "'":
|
|
return this.readString(c, pos);
|
|
}
|
|
|
|
if (isIdentStart(c)) {
|
|
let v = "";
|
|
while (this.pos < src.length && isIdentPart(src[this.pos]!)) v += src[this.pos++];
|
|
return { type: "ident", value: v, pos };
|
|
}
|
|
|
|
throw new LexError(`Unexpected character '${c}' at offset ${pos} (line ${this.lineAt(pos)})`);
|
|
}
|
|
|
|
/** Look at the next token without consuming it. */
|
|
peek(): Token {
|
|
const save = this.pos;
|
|
const t = this.next();
|
|
this.pos = save;
|
|
return t;
|
|
}
|
|
|
|
private readString(quote: string, pos: number): Token {
|
|
const { src } = this;
|
|
let v = "";
|
|
this.pos++; // opening quote
|
|
while (this.pos < src.length) {
|
|
const c = src[this.pos++]!;
|
|
if (c === "\\") {
|
|
const n = src[this.pos++]!;
|
|
v += n === "n" ? "\n" : n === "t" ? "\t" : n;
|
|
continue;
|
|
}
|
|
if (c === quote) return { type: "string", value: v, pos };
|
|
v += c;
|
|
}
|
|
throw new LexError(`Unterminated string at offset ${pos}`);
|
|
}
|
|
|
|
/** Read a route path like `/users/[id]` up to whitespace or `{`. */
|
|
readPath(): string {
|
|
this.skipTrivia();
|
|
const { src } = this;
|
|
let v = "";
|
|
while (this.pos < src.length && !isWs(src[this.pos]!) && src[this.pos] !== "{") {
|
|
v += src[this.pos++];
|
|
}
|
|
if (!v) throw new LexError(`Expected a path at offset ${this.pos}`);
|
|
return v;
|
|
}
|
|
|
|
/**
|
|
* Read a prop default initializer. The initializer may contain nested arrays,
|
|
* objects, calls, strings, or template literals. At top level it ends at a
|
|
* newline, the closing brace of the props block, or the next inline prop
|
|
* declaration (`name = ...` / `name: Type = ...`).
|
|
*/
|
|
readPropInitializer(): string {
|
|
const { src } = this;
|
|
while (this.pos < src.length && (src[this.pos] === " " || src[this.pos] === "\t")) {
|
|
this.pos++;
|
|
}
|
|
|
|
const start = this.pos;
|
|
let square = 0;
|
|
let brace = 0;
|
|
let paren = 0;
|
|
let angle = 0;
|
|
let quote: string | null = null;
|
|
|
|
const atTopLevel = () => square === 0 && brace === 0 && paren === 0 && angle === 0;
|
|
|
|
while (this.pos < src.length) {
|
|
const c = src[this.pos]!;
|
|
|
|
if (quote) {
|
|
this.pos++;
|
|
if (c === "\\" && this.pos < src.length) {
|
|
this.pos++;
|
|
} else if (c === quote) {
|
|
quote = null;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (c === '"' || c === "'" || c === "`") {
|
|
quote = c;
|
|
this.pos++;
|
|
continue;
|
|
}
|
|
|
|
if (atTopLevel()) {
|
|
if (c === "\n" || c === "\r" || c === "}") break;
|
|
|
|
if (c === " " || c === "\t") {
|
|
let look = this.pos;
|
|
while (look < src.length && (src[look] === " " || src[look] === "\t")) look++;
|
|
const rest = src.slice(look);
|
|
if (/^[A-Za-z_][A-Za-z0-9_]*(?:\s*:[^=\r\n{}]+)?\s*=/.test(rest)) break;
|
|
}
|
|
}
|
|
|
|
if (c === "[") square++;
|
|
else if (c === "]" && square > 0) square--;
|
|
else if (c === "{") brace++;
|
|
else if (c === "}" && brace > 0) brace--;
|
|
else if (c === "(") paren++;
|
|
else if (c === ")" && paren > 0) paren--;
|
|
else if (c === "<") angle++;
|
|
else if (c === ">" && angle > 0) angle--;
|
|
|
|
this.pos++;
|
|
}
|
|
|
|
const value = src.slice(start, this.pos).trim();
|
|
if (!value) throw new LexError(`Expected a prop initializer at offset ${start}`);
|
|
return value;
|
|
}
|
|
|
|
/** Read the rest of the current line (used for `state x = <expr>`). */
|
|
readToLineEnd(): string {
|
|
const { src } = this;
|
|
let v = "";
|
|
while (this.pos < src.length && src[this.pos] !== "\n") v += src[this.pos++];
|
|
return v.trim();
|
|
}
|
|
|
|
/**
|
|
* Read a TypeScript-style type annotation after `:`. Reading stops at a
|
|
* top-level `=` or line ending, while nested object/tuple/generic syntax is
|
|
* preserved. The optional `=` is consumed for the caller.
|
|
*/
|
|
readTypeAnnotation(): { type: string; hasDefault: boolean } {
|
|
const { src } = this;
|
|
let value = "";
|
|
let angle = 0;
|
|
let square = 0;
|
|
let brace = 0;
|
|
let paren = 0;
|
|
let quote: string | null = null;
|
|
|
|
while (this.pos < src.length) {
|
|
const c = src[this.pos]!;
|
|
if (quote) {
|
|
value += c;
|
|
this.pos++;
|
|
if (c === "\\" && this.pos < src.length) value += src[this.pos++]!;
|
|
else if (c === quote) quote = null;
|
|
continue;
|
|
}
|
|
if (c === '"' || c === "'" || c === "`") {
|
|
quote = c;
|
|
value += c;
|
|
this.pos++;
|
|
continue;
|
|
}
|
|
if (c === "}" && angle === 0 && square === 0 && brace === 0 && paren === 0) break;
|
|
if (c === "<") angle++;
|
|
else if (c === ">" && angle > 0) angle--;
|
|
else if (c === "[") square++;
|
|
else if (c === "]" && square > 0) square--;
|
|
else if (c === "{") brace++;
|
|
else if (c === "}" && brace > 0) brace--;
|
|
else if (c === "(") paren++;
|
|
else if (c === ")" && paren > 0) paren--;
|
|
|
|
if (angle === 0 && square === 0 && brace === 0 && paren === 0) {
|
|
if (c === " " || c === "\t") {
|
|
let look = this.pos;
|
|
while (look < src.length && (src[look] === " " || src[look] === "\t")) look++;
|
|
if (/^[A-Za-z_][A-Za-z0-9_]*\??\s*:/.test(src.slice(look))) break;
|
|
}
|
|
if (c === "=") {
|
|
this.pos++;
|
|
const type = value.trim();
|
|
if (!type) throw new LexError(`Expected a type annotation at offset ${this.pos}`);
|
|
return { type, hasDefault: true };
|
|
}
|
|
if (c === "\n" || c === "\r") break;
|
|
}
|
|
value += c;
|
|
this.pos++;
|
|
}
|
|
|
|
const type = value.trim();
|
|
if (!type) throw new LexError(`Expected a type annotation at offset ${this.pos}`);
|
|
return { type, hasDefault: false };
|
|
}
|
|
|
|
/**
|
|
* Read a `{ ... }` block and return its INNER text (no outer braces), with
|
|
* brace counting that respects string and template literals so a `}` inside a
|
|
* string doesn't end the block early.
|
|
*/
|
|
readBalancedBraces(): string {
|
|
this.skipTrivia();
|
|
const { src } = this;
|
|
if (src[this.pos] !== "{") {
|
|
throw new LexError(`Expected '{' at offset ${this.pos}`);
|
|
}
|
|
const start = this.pos + 1;
|
|
let depth = 0;
|
|
let i = this.pos;
|
|
let str: string | null = null;
|
|
for (; i < src.length; i++) {
|
|
const c = src[i]!;
|
|
if (str) {
|
|
if (c === "\\") {
|
|
i++;
|
|
continue;
|
|
}
|
|
if (c === str) str = null;
|
|
continue;
|
|
}
|
|
if (c === '"' || c === "'" || c === "`") {
|
|
str = c;
|
|
continue;
|
|
}
|
|
if (c === "{") depth++;
|
|
else if (c === "}") {
|
|
depth--;
|
|
if (depth === 0) {
|
|
this.pos = i + 1;
|
|
return src.slice(start, i);
|
|
}
|
|
}
|
|
}
|
|
throw new LexError(`Unbalanced braces starting at offset ${this.pos}`);
|
|
}
|
|
|
|
private lineAt(pos: number): number {
|
|
let line = 1;
|
|
for (let i = 0; i < pos && i < this.src.length; i++) {
|
|
if (this.src[i] === "\n") line++;
|
|
}
|
|
return line;
|
|
}
|
|
}
|