WRNexusJS/packages/compiler/src/tokenizer.ts

/**
 * Lexer for the `.wrn` language.
 *
 * `.wrn` mixes a small structural grammar (page/state/view/api/realtime) with
 * raw JavaScript bodies. A pure token stream can't represent the raw JS, so the
 * lexer is driven on demand by the parser: it yields structural tokens via
 * `next()`/`peek()`, and exposes `readBalancedBraces()`, `readPath()` and
 * `readToLineEnd()` for the parser to grab raw spans when grammar demands it.
 */

export type TokenType =
  "ident" | "string" | "lbrace" | "rbrace" | "lparen" | "rparen" | "at" | "eq" | "comma" | "eof";

export interface Token {
  type: TokenType;
  value: string;
  pos: number;
}

export class LexError extends Error {}

const isWs = (c: string) => c === " " || c === "\t" || c === "\n" || c === "\r";
const isIdentStart = (c: string) => /[A-Za-z_]/.test(c);
const isIdentPart = (c: string) => /[A-Za-z0-9_]/.test(c);

export class Lexer {
  pos = 0;
  constructor(public readonly src: string) {}

  /** Skip whitespace and `// line comments`. */
  private skipTrivia(): void {
    const { src } = this;
    while (this.pos < src.length) {
      const c = src[this.pos]!;
      if (isWs(c)) {
        this.pos++;
        continue;
      }
      if (c === "/" && src[this.pos + 1] === "/") {
        while (this.pos < src.length && src[this.pos] !== "\n") this.pos++;
        continue;
      }
      break;
    }
  }

  /** Read and consume the next structural token. */
  next(): Token {
    this.skipTrivia();
    const { src } = this;
    const pos = this.pos;
    if (pos >= src.length) return { type: "eof", value: "", pos };

    const c = src[pos]!;
    switch (c) {
      case "{":
        this.pos++;
        return { type: "lbrace", value: c, pos };
      case "}":
        this.pos++;
        return { type: "rbrace", value: c, pos };
      case "(":
        this.pos++;
        return { type: "lparen", value: c, pos };
      case ")":
        this.pos++;
        return { type: "rparen", value: c, pos };
      case "@":
        this.pos++;
        return { type: "at", value: c, pos };
      case "=":
        this.pos++;
        return { type: "eq", value: c, pos };
      case ",":
        this.pos++;
        return { type: "comma", value: c, pos };
      case '"':
      case "'":
        return this.readString(c, pos);
    }

    if (isIdentStart(c)) {
      let v = "";
      while (this.pos < src.length && isIdentPart(src[this.pos]!)) v += src[this.pos++];
      return { type: "ident", value: v, pos };
    }

    throw new LexError(`Unexpected character '${c}' at offset ${pos} (line ${this.lineAt(pos)})`);
  }

  /** Look at the next token without consuming it. */
  peek(): Token {
    const save = this.pos;
    const t = this.next();
    this.pos = save;
    return t;
  }

  private readString(quote: string, pos: number): Token {
    const { src } = this;
    let v = "";
    this.pos++; // opening quote
    while (this.pos < src.length) {
      const c = src[this.pos++]!;
      if (c === "\\") {
        const n = src[this.pos++]!;
        v += n === "n" ? "\n" : n === "t" ? "\t" : n;
        continue;
      }
      if (c === quote) return { type: "string", value: v, pos };
      v += c;
    }
    throw new LexError(`Unterminated string at offset ${pos}`);
  }

  /** Read a route path like `/users/[id]` up to whitespace or `{`. */
  readPath(): string {
    this.skipTrivia();
    const { src } = this;
    let v = "";
    while (this.pos < src.length && !isWs(src[this.pos]!) && src[this.pos] !== "{") {
      v += src[this.pos++];
    }
    if (!v) throw new LexError(`Expected a path at offset ${this.pos}`);
    return v;
  }

  /** Read the rest of the current line (used for `state x = <expr>`). */
  readToLineEnd(): string {
    const { src } = this;
    let v = "";
    while (this.pos < src.length && src[this.pos] !== "\n") v += src[this.pos++];
    return v.trim();
  }

  /**
   * Read a `{ ... }` block and return its INNER text (no outer braces), with
   * brace counting that respects string and template literals so a `}` inside a
   * string doesn't end the block early.
   */
  readBalancedBraces(): string {
    this.skipTrivia();
    const { src } = this;
    if (src[this.pos] !== "{") {
      throw new LexError(`Expected '{' at offset ${this.pos}`);
    }
    const start = this.pos + 1;
    let depth = 0;
    let i = this.pos;
    let str: string | null = null;
    for (; i < src.length; i++) {
      const c = src[i]!;
      if (str) {
        if (c === "\\") {
          i++;
          continue;
        }
        if (c === str) str = null;
        continue;
      }
      if (c === '"' || c === "'" || c === "`") {
        str = c;
        continue;
      }
      if (c === "{") depth++;
      else if (c === "}") {
        depth--;
        if (depth === 0) {
          this.pos = i + 1;
          return src.slice(start, i);
        }
      }
    }
    throw new LexError(`Unbalanced braces starting at offset ${this.pos}`);
  }

  private lineAt(pos: number): number {
    let line = 1;
    for (let i = 0; i < pos && i < this.src.length; i++) {
      if (this.src[i] === "\n") line++;
    }
    return line;
  }
}