wesl 0.7.26 → 0.7.28

package/src/parse/WeslStream.ts

@@ -1,89 +1,51 @@
 import { ParseError } from "../ParseError.ts";
 import type { Stream, TypedToken } from "../Stream.ts";
 import { keywords, reservedWords } from "./Keywords.ts";
-import { CachingStream } from "./stream/CachingStream.ts";
-import { MatchersStream, RegexMatchers } from "./stream/MatchersStream.ts";
-import { matchOneOf } from "./stream/RegexHelpers.ts";
+import { type InternalTokenKind, WeslLexer } from "./stream/WeslLexer.ts";
 export type WeslTokenKind = "word" | "keyword" | "number" | "symbol";
 
 export type WeslToken<Kind extends WeslTokenKind = WeslTokenKind> =
   TypedToken<Kind>;
 
-// https://www.w3.org/TR/WGSL/#blankspace-and-line-breaks
-/** Whitespaces including new lines */
-const blankspaces = /[ \t\n\v\f\r\u{0085}\u{200E}\u{200F}\u{2028}\u{2029}]+/u;
-const symbolSet =
-  "& && -> @ / ! [ ] { } :: : , == = != >>= >> >= > <<= << <= < % - --" +
-  " . + ++ | || ( ) ; * ~ ^ // /* */ += -= *= /= %= &= |= ^=" +
-  // For the _ = expr; syntax
-  " _";
-
-const ident =
-  /(?:(?:[_\p{XID_Start}][\p{XID_Continue}]+)|(?:[\p{XID_Start}]))/u;
-
-/** Checks if a word is a valid WGSL ident, and not a keyword */
-export function isIdent(text: string): boolean {
-  if (text.match(ident)?.[0] !== text) {
-    return false;
-  }
-  if (keywordOrReserved.has(text)) {
-    return false;
-  }
-  return true;
+/** A comment skipped by the tokenizer, recorded as leading trivia of the next token.
+ *  Line-break flags (newline/blank before) are not stored here: they are derived
+ *  on demand during comment attachment, so the tokenizer hot path never counts
+ *  line breaks. */
+export interface CommentTrivia {
+  style: "line" | "block";
+  /** Source range of the comment text (excluding the trailing newline of a line comment). */
+  start: number;
+  end: number;
 }
 
-const keywordOrReserved = new Set(keywords.concat(reservedWords));
-
-const digits = new RegExp(
-  // decimal_float_literal
-  /(?:0[fh])|(?:[1-9][0-9]*[fh])/.source +
-    /|(?:[0-9]*\.[0-9]+(?:[eE][+-]?[0-9]+)?[fh]?)/.source +
-    /|(?:[0-9]+\.[0-9]*(?:[eE][+-]?[0-9]+)?[fh]?)/.source +
-    /|(?:[0-9]+[eE][+-]?[0-9]+[fh]?)/.source +
-    // hex_float_literal
-    /|(?:0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+(?:[pP][+-]?[0-9]+[fh]?)?)/.source +
-    /|(?:0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*(?:[pP][+-]?[0-9]+[fh]?)?)/.source +
-    /|(?:0[xX][0-9a-fA-F]+[pP][+-]?[0-9]+[fh]?)/.source +
-    // hex_int_literal
-    /|(?:0[xX][0-9a-fA-F]+[iu]?)/.source +
-    // decimal_int_literal
-    /|(?:0[iu]?)|(?:[1-9][0-9]*[iu]?)/.source,
-);
+/** One line break, treating \r\n as a single break.
+ *  Same code points as `isLineBreak` in AttachComments.ts (charCode form). */
+const lineBreak = String.raw`\r\n?|[\n\v\f\u{0085}\u{2028}\u{2029}]`;
 
-const commentStart = /\/\/|\/\*/;
-
-type InternalTokenKind =
-  | "word"
-  | "number"
-  | "blankspaces"
-  | "commentStart"
-  | "symbol"
-  | "invalid";
-const weslMatcher = new RegexMatchers<InternalTokenKind>({
-  word: ident,
-  number: digits,
-  blankspaces,
-  commentStart,
-  symbol: matchOneOf(symbolSet),
-  // biome-ignore lint/correctness/noEmptyCharacterClassInRegex: TODO
-  invalid: /[^]/,
-});
+const keywordOrReserved = new Set(keywords.concat(reservedWords));
 
-/** To mark parts of the grammar implementation that are WESL specific extensions */
-export function weslExtension<T>(combinator: T): T {
-  return combinator;
+/** A peeked token cached at the position it was read from. */
+interface PeekedToken {
+  pos: number;
+  token: WeslToken | null;
+  end: number;
 }
 
 /** A stream that produces WESL tokens, skipping over comments and white space */
 export class WeslStream implements Stream<WeslToken> {
   private stream: Stream<TypedToken<InternalTokenKind>>;
-  /** New line */
-  private eolPattern = /[\n\v\f\u{0085}\u{2028}\u{2029}]|\r\n?/gu;
+  /** New line (stateful: scanned via lastIndex, so kept per-instance). */
+  private eolPattern = new RegExp(lineBreak, "gu");
   private blockCommentPattern = /\/\*|\*\//g;
+  /** Comments skipped before a real token, keyed by that token's start position. */
+  private triviaByPos = new Map<number, CommentTrivia[]>();
+  /** Last peeked token, so the following nextToken() skips the rescan.
+   *  Never invalidated: tokenization is deterministic per position. */
+  private peeked: PeekedToken | null = null;
   public src: string;
   constructor(src: string) {
     this.src = src;
-    this.stream = new CachingStream(new MatchersStream(src, weslMatcher));
+    this.stream = new WeslLexer(src);
   }
   checkpoint(): number {
     return this.stream.checkpoint();
@@ -91,40 +53,87 @@ export class WeslStream implements Stream<WeslToken> {
   reset(position: number): void {
     this.stream.reset(position);
   }
+  /** All recorded comment runs (each a contiguous group of comments between two
+   *  real tokens), in source order. Consumed by the post-parse comment pass. */
+  commentRuns(): CommentTrivia[][] {
+    return [...this.triviaByPos.entries()]
+      .sort((a, b) => a[0] - b[0])
+      .map(([, run]) => run);
+  }
+  private recordTrivia(pos: number, pending?: CommentTrivia[]): void {
+    if (pending) this.triviaByPos.set(pos, pending);
+  }
+
+  /** Next real token (comments/blankspace skipped and recorded as trivia); null at EOF. */
   nextToken(): WeslToken | null {
+    const peeked = this.usePeeked();
+    if (peeked !== null) return peeked.token;
+
+    let pending: CommentTrivia[] | undefined;
     while (true) {
       const token = this.stream.nextToken();
-      if (token === null) return null;
+      if (token === null) {
+        // trailing comments at end of file: key them past the last position
+        this.recordTrivia(this.src.length, pending);
+        return null;
+      }
 
       const kind = token.kind;
       if (kind === "blankspaces") {
-        continue;
+        continue; // newline/blank flags are derived later, at attach time
       } else if (kind === "commentStart") {
-        // SAFETY: The underlying streams can be seeked to any position
-        if (token.text === "//") {
-          this.stream.reset(this.skipToEol(token.span[1]));
-        } else {
-          this.stream.reset(this.skipBlockComment(token.span[1]));
-        }
-      } else if (kind === "word") {
-        const returnToken = token as TypedToken<typeof kind | "keyword">;
-        if (keywordOrReserved.has(token.text)) {
-          returnToken.kind = "keyword";
-        }
-        return returnToken;
+        pending ??= [];
+        pending.push(this.consumeComment(token));
       } else if (kind === "invalid") {
         throw new ParseError("Invalid token " + token.text, token.span);
       } else {
-        return token as TypedToken<typeof kind>;
+        this.recordTrivia(token.span[0], pending);
+        const result = token as WeslToken;
+        if (kind === "word" && keywordOrReserved.has(token.text)) {
+          result.kind = "keyword";
+        }
+        return result;
       }
     }
   }
 
+  /** If the last peek() was at the current position, consume and return it. */
+  private usePeeked(): PeekedToken | null {
+    const peeked = this.peeked;
+    if (peeked !== null && peeked.pos === this.checkpoint()) {
+      this.reset(peeked.end);
+      return peeked;
+    }
+    return null;
+  }
+
+  /** Skip a comment (rejecting embedded \0), advance the stream past it,
+   *  and return it as trivia. */
+  private consumeComment(token: TypedToken<InternalTokenKind>): CommentTrivia {
+    const style = token.text === "//" ? "line" : "block";
+    const start = token.span[0];
+    const end = this.commentEnd(token);
+    // WGSL forbids the null code point anywhere, including inside comments
+    // (a comment body is skipped here, so the `invalid` matcher never sees
+    // it). Scan only the comment body, keeping this O(comment length).
+    const bodyNull = this.src.slice(start, end).indexOf("\0");
+    if (bodyNull >= 0) {
+      const at = start + bodyNull;
+      throw new ParseError("Invalid token \\0", [at, at + 1]);
+    }
+    this.stream.reset(end);
+    return { style, start, end };
+  }
+
   /** Peek at the next token without consuming it */
   peek(): WeslToken | null {
     const pos = this.checkpoint();
+    const peeked = this.peeked;
+    if (peeked !== null && peeked.pos === pos) return peeked.token;
     const token = this.nextToken();
+    const end = this.checkpoint();
     this.reset(pos);
+    this.peeked = { pos, token, end };
     return token;
   }
 
@@ -176,16 +185,18 @@ export class WeslStream implements Stream<WeslToken> {
     return tokens;
   }
 
-  private skipToEol(position: number): number {
+  /** End position of a comment opened by a commentStart token. */
+  private commentEnd(token: TypedToken<InternalTokenKind>): number {
+    return token.text === "//"
+      ? this.lineCommentEnd(token.span[1])
+      : this.skipBlockComment(token.span[1]);
+  }
+
+  /** End of a line comment: the start of the next line break (or end of file). */
+  private lineCommentEnd(position: number): number {
     this.eolPattern.lastIndex = position;
     const result = this.eolPattern.exec(this.src);
-    if (result === null) {
-      // We reached the end of the file
-      return this.src.length;
-    } else {
-      // Move forward
-      return this.eolPattern.lastIndex;
-    }
+    return result === null ? this.src.length : result.index;
   }
 
   private skipBlockComment(start: number): number {
@@ -196,10 +207,9 @@ export class WeslStream implements Stream<WeslToken> {
       if (result === null) {
         throw new ParseError("Unclosed block comment!", [position, position]);
       } else if (result[0] === "*/") {
-        // Close block
         return this.blockCommentPattern.lastIndex;
       } else if (result[0] === "/*") {
-        // Open block
+        // nested block comment: recurse so its */ doesn't close the outer one
         position = this.skipBlockComment(this.blockCommentPattern.lastIndex);
       } else {
         throw new Error("Unreachable, invalid block comment pattern");
@@ -214,58 +224,66 @@ export class WeslStream implements Stream<WeslToken> {
    */
   nextTemplateStartToken(): (WeslToken & { kind: "symbol" }) | null {
     const startPosition = this.stream.checkpoint();
-    const token: WeslToken | null = this.nextToken();
+    const token = this.nextToken();
     this.stream.reset(startPosition);
-    if (token === null) return null;
 
-    if (token.kind !== "symbol") {
+    //<<= << <= cannot be templates, so we match the entire token text
+    if (token === null || token.kind !== "symbol" || token.text !== "<") {
       return null;
     }
-
-    //<<= << <= cannot be templates, so we match the entire token text
-    if (token.text === "<") {
-      if (this.isTemplateStart(token.span[1])) {
-        this.stream.reset(token.span[1]);
-        return token as WeslToken & { kind: typeof token.kind };
-      } else {
-        this.stream.reset(startPosition);
-        return null;
-      }
-    } else {
+    if (!this.isTemplateStart(token.span[1])) {
+      this.stream.reset(startPosition); // isTemplateStart advanced the stream
       return null;
     }
+    this.stream.reset(token.span[1]);
+    return token as WeslToken & { kind: "symbol" };
   }
 
+  /** Match a template-closing `>`, splitting it off a `>>`/`>=`/`>>=` token when needed. */
   nextTemplateEndToken(): (WeslToken & { kind: "symbol" }) | null {
     const startPosition = this.stream.checkpoint();
-    const token: WeslToken | null = this.nextToken();
+    const token = this.nextToken();
     this.stream.reset(startPosition);
     if (token === null) return null;
 
-    // template closing can also match a >= or >>, so we split the token
-    if (token.kind === "symbol" && token.text[0] === ">") {
-      // SAFETY: The underlying streams implementations can be reset to any position.
-      const tokenPosition = token.span[0];
-      this.stream.reset(tokenPosition + 1);
-      return {
-        kind: "symbol",
-        span: [tokenPosition, tokenPosition + 1],
-        text: ">",
-      };
-    } else {
-      return null;
+    // Template closing can also match a >= or >> here, so split one `>` off
+    // and re-lex the rest. Safe for the closed set of `>`-leading symbols in
+    // WeslLexer.ts (`>` `>=` `>>` `>>=`): each remainder (``, `=`, `>`, `>=`)
+    // re-lexes to the intended tokens. A new `>`-leading symbol would break
+    // this split; isTemplateStart's exhaustive check throws if the set grows.
+    if (token.kind !== "symbol" || token.text[0] !== ">") return null;
+
+    // SAFETY: The underlying streams implementations can be reset to any position.
+    const tokenPosition = token.span[0];
+    this.stream.reset(tokenPosition + 1);
+    return {
+      kind: "symbol",
+      span: [tokenPosition, tokenPosition + 1],
+      text: ">",
+    };
+  }
+
+  /** Next symbol from the raw stream, skipping comment bodies (so symbols
+   *  inside comments don't count) and non-symbol tokens; null at EOF. */
+  private nextRawSymbol(): TypedToken<InternalTokenKind> | null {
+    while (true) {
+      const token = this.stream.nextToken();
+      if (token === null) return null;
+      if (token.kind === "commentStart") {
+        this.stream.reset(this.commentEnd(token));
+      } else if (token.kind === "symbol") return token;
     }
   }
 
+  /** Template-list discovery: scan forward from `<` for a balanced closing `>`. */
   private isTemplateStart(afterToken: number): boolean {
     // Skip over <
     this.stream.reset(afterToken);
     // We start with a < token
     let pendingCounter = 1;
     while (true) {
-      const nextToken = this.stream.nextToken();
+      const nextToken = this.nextRawSymbol();
       if (nextToken === null) return false;
-      if (nextToken.kind !== "symbol") continue;
       if (nextToken.text === "<") {
         // Start a nested template
         pendingCounter += 1;
@@ -306,12 +324,11 @@ export class WeslStream implements Stream<WeslToken> {
    */
   private skipBracketsTo(closingBracket: string): void {
     while (true) {
-      const nextToken = this.stream.nextToken();
+      const nextToken = this.nextRawSymbol();
       if (nextToken === null) {
         const after = this.stream.checkpoint();
         throw new ParseError("Unclosed bracket!", [after, after]);
       }
-      if (nextToken.kind !== "symbol") continue;
       if (nextToken.text === "(") {
         this.skipBracketsTo(")");
       } else if (nextToken.text === "[") {

package/src/parse/stream/RegexMatchers.ts

@@ -0,0 +1,45 @@
+import type { Span } from "../../Span.ts";
+import type { TypedToken } from "../../Stream.ts";
+import { toRegexSource } from "./RegexHelpers.ts";
+
+/**
+ * Matches tokens by kind with one combined regex.
+ *
+ * The matchers passed to this object must follow certain rules:
+ * - They must use non-capturing groups: `(?:...)`
+ * - They must NOT use `^` or `$`
+ */
+export class RegexMatchers<Kind extends string> {
+  private groups: Kind[];
+  private exp: RegExp;
+  constructor(matchers: Record<Kind, string | RegExp>) {
+    this.groups = Object.keys(matchers) as Kind[];
+    const expParts = Object.entries(matchers as Record<string, string | RegExp>)
+      .map(toRegexSource)
+      .join("|");
+    // y = sticky, match exactly at lastIndex (set per call in execAt)
+    // u = unicode aware
+    this.exp = new RegExp(expParts, "yu");
+  }
+
+  execAt(text: string, position: number): TypedToken<Kind> | null {
+    this.exp.lastIndex = position;
+    const matches = this.exp.exec(text);
+    if (matches === null) return null;
+
+    // each matcher is a full-token capturing group, so the matched substring
+    // is the whole match and the span starts at the sticky position
+    const matched = matches[0];
+    const span: Span = [position, position + matched.length];
+    const kind = this.groups[matchedGroupIndex(matches)];
+    return { kind, span, text: matched };
+  }
+}
+
+/** @return index of the alternation group that matched */
+function matchedGroupIndex(matches: RegExpExecArray): number {
+  for (let i = 1; i < matches.length; i++) {
+    if (matches[i] !== undefined) return i - 1;
+  }
+  throw new Error("no matching group");
+}

package/src/parse/stream/WeslLexer.ts

@@ -0,0 +1,249 @@
+import type { Stream, TypedToken } from "../../Stream.ts";
+import { matchOneOf } from "./RegexHelpers.ts";
+import { RegexMatchers } from "./RegexMatchers.ts";
+
+export type InternalTokenKind =
+  | "word"
+  | "number"
+  | "blankspaces"
+  | "commentStart"
+  | "symbol"
+  | "invalid";
+
+type RawToken = TypedToken<InternalTokenKind>;
+
+const ident =
+  /(?:(?:[_\p{XID_Start}][\p{XID_Continue}]+)|(?:[\p{XID_Start}]))/u;
+
+// https://www.w3.org/TR/WGSL/#blankspace-and-line-breaks
+/** Whitespaces including new lines */
+const blankspaces = /[ \t\n\v\f\r\u{0085}\u{200E}\u{200F}\u{2028}\u{2029}]+/u;
+
+const digits = new RegExp(
+  // decimal_float_literal
+  /(?:0[fh])|(?:[1-9][0-9]*[fh])/.source +
+    /|(?:[0-9]*\.[0-9]+(?:[eE][+-]?[0-9]+)?[fh]?)/.source +
+    /|(?:[0-9]+\.[0-9]*(?:[eE][+-]?[0-9]+)?[fh]?)/.source +
+    /|(?:[0-9]+[eE][+-]?[0-9]+[fh]?)/.source +
+    // hex_float_literal
+    /|(?:0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+(?:[pP][+-]?[0-9]+[fh]?)?)/.source +
+    /|(?:0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*(?:[pP][+-]?[0-9]+[fh]?)?)/.source +
+    /|(?:0[xX][0-9a-fA-F]+[pP][+-]?[0-9]+[fh]?)/.source +
+    // hex_int_literal
+    /|(?:0[xX][0-9a-fA-F]+[iu]?)/.source +
+    // decimal_int_literal
+    /|(?:0[iu]?)|(?:[1-9][0-9]*[iu]?)/.source,
+);
+
+const symbolSet =
+  "& && -> @ / ! [ ] { } :: : , == = != >>= >> >= > <<= << <= < % - --" +
+  " . + ++ | || ( ) ; * ~ ^ // /* */ += -= *= /= %= &= |= ^=" +
+  // For the _ = expr; syntax
+  " _";
+
+const commentStart = /\/\/|\/\*/;
+
+/** Unicode-aware fallback for characters the ASCII fast path can't handle.
+ *  lastIndex is set on every use, so sharing one instance is safe. */
+const unicodeMatcher = new RegexMatchers<InternalTokenKind>({
+  word: ident,
+  number: digits,
+  blankspaces,
+  commentStart,
+  symbol: matchOneOf(symbolSet),
+  // biome-ignore lint/correctness/noEmptyCharacterClassInRegex: [^] deliberately matches any char (incl. newlines) as the catch-all
+  invalid: /[^]/,
+});
+
+/**
+ * Hand-written scanner for WESL/WGSL raw tokens.
+ *
+ * The common case is dispatched on ASCII char codes: whitespace is skipped
+ * without producing tokens, and idents, numbers, symbols, and comment openers
+ * are matched directly. Any character >= 0x80 (unicode idents, unicode
+ * blankspace) falls back to the unicode-aware regex, so there is no up-front
+ * scan and no unicode restriction.
+ */
+export class WeslLexer implements Stream<RawToken> {
+  private position = 0;
+  /** sticky number matcher (stateful via lastIndex, so kept per-instance) */
+  private numberExp = new RegExp(digits.source, "y");
+  public src: string;
+
+  constructor(src: string) {
+    this.src = src;
+  }
+
+  checkpoint(): number {
+    return this.position;
+  }
+  reset(position: number): void {
+    this.position = position;
+  }
+
+  nextToken(): RawToken | null {
+    const { src } = this;
+    const len = src.length;
+    let pos = this.position;
+    // skip ASCII blankspace; unicode blankspace falls through to the regex
+    while (pos < len) {
+      const c = src.charCodeAt(pos);
+      if (c === 0x20 || (c >= 0x09 && c <= 0x0d)) pos++;
+      else break;
+    }
+    if (pos >= len) {
+      this.position = pos;
+      return null;
+    }
+
+    const c = src.charCodeAt(pos);
+    if ((c >= 0x61 && c <= 0x7a) || (c >= 0x41 && c <= 0x5a)) {
+      return this.word(pos);
+    }
+    if (c >= 0x30 && c <= 0x39) return this.number(pos);
+
+    // n is NaN past the end of source; all comparisons with NaN are false
+    const n = src.charCodeAt(pos + 1);
+    switch (c) {
+      case 0x5f: // _
+        if (isWordChar(n)) return this.word(pos);
+        if (n >= 0x80) return this.regexToken(pos); // _ + unicode continue
+        return this.symbol(pos, "_");
+      case 0x2f: // /
+        if (n === 0x2f) return this.token("commentStart", pos, "//");
+        if (n === 0x2a) return this.token("commentStart", pos, "/*");
+        if (n === 0x3d) return this.symbol(pos, "/=");
+        return this.symbol(pos, "/");
+      case 0x2e: // .
+        if (n >= 0x30 && n <= 0x39) return this.number(pos); // .5 float
+        return this.symbol(pos, ".");
+      case 0x3e: // >
+        if (n === 0x3e) {
+          if (src.charCodeAt(pos + 2) === 0x3d) return this.symbol(pos, ">>=");
+          return this.symbol(pos, ">>");
+        }
+        if (n === 0x3d) return this.symbol(pos, ">=");
+        return this.symbol(pos, ">");
+      case 0x3c: // <
+        if (n === 0x3c) {
+          if (src.charCodeAt(pos + 2) === 0x3d) return this.symbol(pos, "<<=");
+          return this.symbol(pos, "<<");
+        }
+        if (n === 0x3d) return this.symbol(pos, "<=");
+        return this.symbol(pos, "<");
+      case 0x26: // &
+        if (n === 0x26) return this.symbol(pos, "&&");
+        if (n === 0x3d) return this.symbol(pos, "&=");
+        return this.symbol(pos, "&");
+      case 0x7c: // |
+        if (n === 0x7c) return this.symbol(pos, "||");
+        if (n === 0x3d) return this.symbol(pos, "|=");
+        return this.symbol(pos, "|");
+      case 0x2d: // -
+        if (n === 0x3e) return this.symbol(pos, "->");
+        if (n === 0x2d) return this.symbol(pos, "--");
+        if (n === 0x3d) return this.symbol(pos, "-=");
+        return this.symbol(pos, "-");
+      case 0x2b: // +
+        if (n === 0x2b) return this.symbol(pos, "++");
+        if (n === 0x3d) return this.symbol(pos, "+=");
+        return this.symbol(pos, "+");
+      case 0x3a: // :
+        if (n === 0x3a) return this.symbol(pos, "::");
+        return this.symbol(pos, ":");
+      case 0x3d: // =
+        if (n === 0x3d) return this.symbol(pos, "==");
+        return this.symbol(pos, "=");
+      case 0x21: // !
+        if (n === 0x3d) return this.symbol(pos, "!=");
+        return this.symbol(pos, "!");
+      case 0x2a: // *
+        if (n === 0x2f) return this.symbol(pos, "*/");
+        if (n === 0x3d) return this.symbol(pos, "*=");
+        return this.symbol(pos, "*");
+      case 0x25: // %
+        if (n === 0x3d) return this.symbol(pos, "%=");
+        return this.symbol(pos, "%");
+      case 0x5e: // ^
+        if (n === 0x3d) return this.symbol(pos, "^=");
+        return this.symbol(pos, "^");
+      case 0x40: // @
+        return this.symbol(pos, "@");
+      case 0x5b: // [
+        return this.symbol(pos, "[");
+      case 0x5d: // ]
+        return this.symbol(pos, "]");
+      case 0x7b: // {
+        return this.symbol(pos, "{");
+      case 0x7d: // }
+        return this.symbol(pos, "}");
+      case 0x28: // (
+        return this.symbol(pos, "(");
+      case 0x29: // )
+        return this.symbol(pos, ")");
+      case 0x2c: // ,
+        return this.symbol(pos, ",");
+      case 0x3b: // ;
+        return this.symbol(pos, ";");
+      case 0x7e: // ~
+        return this.symbol(pos, "~");
+    }
+    if (c >= 0x80) return this.regexToken(pos);
+    this.position = pos + 1;
+    return { kind: "invalid", span: [pos, pos + 1], text: src[pos] };
+  }
+
+  private token(
+    kind: InternalTokenKind,
+    start: number,
+    text: string,
+  ): RawToken {
+    const end = start + text.length;
+    this.position = end;
+    return { kind, span: [start, end], text };
+  }
+
+  private symbol(start: number, text: string): RawToken {
+    return this.token("symbol", start, text);
+  }
+
+  private word(start: number): RawToken | null {
+    const { src } = this;
+    const len = src.length;
+    let pos = start + 1;
+    while (pos < len) {
+      const c = src.charCodeAt(pos);
+      if (isWordChar(c)) pos++;
+      else if (c >= 0x80)
+        return this.regexToken(start); // unicode ident
+      else break;
+    }
+    this.position = pos;
+    return { kind: "word", span: [start, pos], text: src.slice(start, pos) };
+  }
+
+  private number(start: number): RawToken {
+    this.numberExp.lastIndex = start;
+    // non-null: callers guarantee a leading digit (matches decimal_int) or
+    // a `.` followed by a digit (matches the fraction-first float alternative)
+    const text = this.numberExp.exec(this.src)![0];
+    return this.token("number", start, text);
+  }
+
+  /** match one token (or blankspace run) with the unicode-aware regex */
+  private regexToken(start: number): RawToken | null {
+    const token = unicodeMatcher.execAt(this.src, start);
+    if (token === null) return null; // unreachable: `invalid` matches any char
+    this.position = token.span[1];
+    return token;
+  }
+}
+
+function isWordChar(c: number): boolean {
+  return (
+    (c >= 0x61 && c <= 0x7a) || // a-z
+    (c >= 0x41 && c <= 0x5a) || // A-Z
+    (c >= 0x30 && c <= 0x39) || // 0-9
+    c === 0x5f // _
+  );
+}