parseman 0.17.0 → 0.18.0

package/dist/index.d.ts

@@ -15,6 +15,7 @@ export type { BuildNode, NodeOptions } from './combinators/node.ts';
 export { transform, skip, trivia, label } from './combinators/map.ts';
 export { parse, parser, noTrivia } from './combinators/grammar.ts';
 export type { ParseOptions, ParserOptions, ParsemanParser } from './combinators/grammar.ts';
+export { token } from './combinators/token.ts';
 export { compile } from './compiler/codegen.ts';
 export type { CompiledParser, LinkablePieces } from './compiler/codegen.ts';
 export { compose, pick, cstBuildHost } from './compiler/linker.ts';
@@ -24,7 +25,7 @@ export type { LineIndex } from './compiler/line-index.ts';
 export { guard } from './combinators/guard.ts';
 export { withCtx } from './combinators/withCtx.ts';
 export { recover, isParseError } from './combinators/recover.ts';
-export { expect, staticExpected } from './combinators/expect.ts';
+export { expect } from './combinators/expect.ts';
 export { completionsAt } from './combinators/completions.ts';
 export { scanTo, balanced } from './combinators/scanTo.ts';
 export type { ScanToOptions } from './combinators/scanTo.ts';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAEjM,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,YAAY,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAE9D,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAQ9C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AACpE,YAAY,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAChD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAC1E,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAA;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAA;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAA;AAC5C,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AAEnE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AACrE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAClE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAE3F,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AAClE,YAAY,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAErD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACxF,YAAY,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAA;AAEzD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAChE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAC5D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAC1D,YAAY,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAC5D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAEzE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAE5G,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AACtF,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAA;AACzC,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AACzF,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AACnD,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AACvD,YAAY,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAA;AAEhE,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,WAAW,EACX,WAAW,EACX,UAAU,EACV,gBAAgB,GACjB,MAAM,uBAAuB,CAAA;AAC9B,YAAY,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EACL,UAAU,EACV,UAAU,EACV,cAAc,EACd,aAAa,EACb,SAAS,GACV,MAAM,yBAAyB,CAAA;AAChC,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAEjM,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,YAAY,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAE9D,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAQ9C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AACpE,YAAY,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAChD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAC1E,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAA;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAA;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAA;AAC5C,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AAEnE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AACrE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAClE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC3F,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AAClE,YAAY,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAErD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACxF,YAAY,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAA;AAEzD,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAC5D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAC1D,YAAY,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAC5D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAEzE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAE5G,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AACtF,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAA;AACzC,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AACzF,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AACnD,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AACvD,YAAY,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAA;AAEhE,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,WAAW,EACX,WAAW,EACX,UAAU,EACV,gBAAgB,GACjB,MAAM,uBAAuB,CAAA;AAC9B,YAAY,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EACL,UAAU,EACV,UAAU,EACV,cAAc,EACd,aAAa,EACb,SAAS,GACV,MAAM,yBAAyB,CAAA;AAChC,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA"}

package/dist/index.js

@@ -58,6 +58,7 @@ function matchesEmpty(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "expect":
     case "withCtx":
     case "node":
@@ -120,6 +121,7 @@ function firstSetOf(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "node":
     case "grammar":
     case "expect":
@@ -1246,6 +1248,7 @@ function markUnusedValues(root) {
         return;
       case "label":
       case "trivia":
+      case "token":
       case "withCtx":
       case "expect":
         visit(def.parser, consumed);
@@ -1546,8 +1549,60 @@ function parse(combinator, input, opts = {}) {
   return withErrors;
 }
 
+// src/combinators/token.ts
+function token(root) {
+  const meta = {
+    firstSet: root._meta.firstSet,
+    canMatchNewline: root._meta.canMatchNewline,
+    isTrivia: false
+  };
+  return {
+    _tag: "token",
+    _meta: meta,
+    _def: { tag: "token", parser: root },
+    parse(input, pos, ctx) {
+      const savedTrivia = ctx.trivia;
+      const savedKinds = ctx.triviaKindLabels;
+      const savedBuf = ctx._cstBuf;
+      const savedChildren = ctx._cstChildren;
+      const savedLeaves = ctx._cstLeaves;
+      const savedRaw = ctx._cstRawChildren;
+      const savedTriviaLog = ctx._cstTriviaLog;
+      const savedOuterTriviaLog = ctx._triviaLog;
+      const wasCapturing = cstCaptureActive(ctx);
+      ctx.trivia = void 0;
+      ctx.triviaKindLabels = void 0;
+      ctx._cstBuf = void 0;
+      ctx._cstChildren = void 0;
+      ctx._cstLeaves = void 0;
+      ctx._cstRawChildren = void 0;
+      ctx._cstTriviaLog = void 0;
+      delete ctx._triviaLog;
+      let result;
+      try {
+        result = root.parse(input, pos, ctx);
+      } finally {
+        ctx.trivia = savedTrivia;
+        ctx.triviaKindLabels = savedKinds;
+        ctx._cstBuf = savedBuf;
+        ctx._cstChildren = savedChildren;
+        ctx._cstLeaves = savedLeaves;
+        ctx._cstRawChildren = savedRaw;
+        ctx._cstTriviaLog = savedTriviaLog;
+        if (savedOuterTriviaLog === void 0) delete ctx._triviaLog;
+        else ctx._triviaLog = savedOuterTriviaLog;
+      }
+      if (!result.ok) return result;
+      const span = result.span;
+      const value = input.slice(pos, span.end);
+      if (wasCapturing) pushCstLeaf(ctx, { _tag: "leaf", value, span });
+      return { ok: true, value, span };
+    }
+  };
+}
+
 // src/combinators/expect.ts
-function staticExpected(c) {
+function deriveExpected(c) {
   const def = c._def;
   switch (def.tag) {
     case "literal":
@@ -1559,22 +1614,23 @@ function staticExpected(c) {
     case "label":
       return [def.label];
     case "choice":
-      return def.parsers.flatMap(staticExpected);
+      return def.parsers.flatMap(deriveExpected);
     case "sequence":
-      return def.parsers.length > 0 ? staticExpected(def.parsers[0]) : [];
+      return def.parsers.length > 0 ? deriveExpected(def.parsers[0]) : [];
     case "node":
     case "grammar":
     case "trivia":
+    case "token":
     case "optional":
     case "many":
     case "oneOrMore":
     case "transform":
     case "not":
-      return staticExpected(def.parser);
+      return deriveExpected(def.parser);
     case "lazy": {
       const name = c._ruleName;
       try {
-        return staticExpected(def.thunk());
+        return deriveExpected(def.thunk());
       } catch {
         return name ? [name] : [];
       }
@@ -1584,7 +1640,7 @@ function staticExpected(c) {
   }
 }
 function expect(combinator, label2) {
-  const expected = label2 !== void 0 ? [label2] : staticExpected(combinator);
+  const expected = label2 !== void 0 ? [label2] : deriveExpected(combinator);
   const meta = {
     firstSet: combinator._meta.firstSet,
     canMatchNewline: combinator._meta.canMatchNewline,
@@ -3236,6 +3292,7 @@ function mayLeavePartialCapture(p, seen = /* @__PURE__ */ new Set()) {
     case "guard":
     case "not":
     case "trivia":
+    case "token":
     case "scanTo":
     case "unknown":
       return false;
@@ -3297,6 +3354,7 @@ function capturesLeaf(p, seen = /* @__PURE__ */ new Set()) {
     case "regex":
     case "keywords":
     case "node":
+    case "token":
       return true;
     case "not":
     case "guard":
@@ -3575,9 +3633,9 @@ function emitSeq(def, ctx, pos) {
   stmts.push(`${ind(ctx)}const ${arrV} = [${valueVars.join(", ")}]`);
   return { stmts, valueVar: arrV, endVar };
 }
-function staticExpectedArr(parsers) {
+function deriveExpectedArr(parsers) {
   return JSON.stringify(parsers.flatMap((p) => {
-    const e = staticExpected(p);
+    const e = deriveExpected(p);
     return e.length > 0 ? e : [p._tag];
   }));
 }
@@ -3598,7 +3656,7 @@ function failsAtStart(p) {
   }
 }
 function armStaticExpected(ctx, p) {
-  return hoistExpected(ctx, staticExpectedArr([p]));
+  return hoistExpected(ctx, deriveExpectedArr([p]));
 }
 function canMatchEmptyAtStart(p) {
   const d = p._def;
@@ -3636,7 +3694,7 @@ function emitDisjointArm(p, ctx, pos, valV, endV) {
   ];
 }
 function emitChoice(def, ctx, pos) {
-  const allExpected = staticExpectedArr(def.parsers);
+  const allExpected = deriveExpectedArr(def.parsers);
   if (def.disjoint) {
     const codeV = v(ctx, "_code");
     const valV = v(ctx, "_chv");
@@ -3697,7 +3755,7 @@ function emitGreedyClassify(def, superIndex, ctx, pos) {
   const wordV = v(ctx, "_gw");
   const endV = v(ctx, "_ge");
   const valV = v(ctx, "_gcv");
-  const regexExpected = JSON.stringify(staticExpected(superParser));
+  const regexExpected = JSON.stringify(deriveExpected(superParser));
   const stmts = [
     `${ind(ctx)}${reVar}.lastIndex = ${pos}`,
     `${ind(ctx)}const ${matchV} = ${reVar}.exec(input)`,
@@ -4089,6 +4147,116 @@ function emitNot(def, ctx, pos) {
     endVar: pos
   };
 }
+function escapeTokenLiteral(value) {
+  return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+}
+function regexMatchesEmpty(source, flags) {
+  try {
+    return new RegExp(`^(?:${source})$`, flags).test("");
+  } catch {
+    return true;
+  }
+}
+function tokenTerminalSource(p) {
+  const def = p._def;
+  switch (def.tag) {
+    case "literal":
+      if (def.value.length === 0) return null;
+      return {
+        source: escapeTokenLiteral(def.value),
+        caseInsensitive: def.caseInsensitive,
+        rawCaseSensitive: def.caseInsensitive ? "" : def.value
+      };
+    case "regex": {
+      if (def.flags.replace(/i/g, "") !== "") return null;
+      if (regexMatchesEmpty(def.source, def.flags)) return null;
+      return {
+        source: `(?:${def.source})`,
+        caseInsensitive: def.flags.includes("i"),
+        rawCaseSensitive: def.flags.includes("i") ? "" : def.source
+      };
+    }
+    default:
+      return null;
+  }
+}
+function tokenSequenceSource(p) {
+  const def = p._def;
+  const parts = def.tag === "sequence" ? def.parsers.map(tokenTerminalSource) : [tokenTerminalSource(p)];
+  if (parts.some((part) => part === null)) return null;
+  const typed = parts;
+  const flags = typed.some((part) => part.caseInsensitive) ? "i" : "";
+  if (flags.includes("i") && typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))) return null;
+  return {
+    source: typed.map((part) => part.source).join(""),
+    flags,
+    hasCaseSensitiveLetters: typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))
+  };
+}
+function combineTokenRegexSources(parts) {
+  const flags = parts.some((part) => part.flags.includes("i")) ? "i" : "";
+  if (flags.includes("i") && parts.some((part) => part.flags === "" && part.hasCaseSensitiveLetters)) return null;
+  return { source: parts.map((part) => part.source).join(""), flags };
+}
+function tokenNullableRegex(def) {
+  const inner = def.parser._def;
+  if (inner.tag === "many") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})*`, flags: seq.flags };
+  }
+  if (inner.tag === "optional") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})?`, flags: seq.flags };
+  }
+  if (inner.tag === "sepBy") {
+    const item = tokenSequenceSource(inner.parser);
+    const sep = tokenSequenceSource(inner.separator);
+    if (!item || !sep) return null;
+    const combined = combineTokenRegexSources([item, sep]);
+    if (!combined) return null;
+    return { tag: "regex", source: `(?:${item.source}(?:${sep.source}${item.source})*)?`, flags: combined.flags };
+  }
+  return null;
+}
+function emitToken(def, ctx, pos) {
+  const lowered = tokenNullableRegex(def);
+  if (lowered) return emitRegex(lowered, ctx, pos);
+  const savedTrivia = ctx.activeTrivia;
+  const savedKindLabels = ctx.triviaKindLabels;
+  ctx.activeTrivia = void 0;
+  ctx.triviaKindLabels = void 0;
+  const inner = emitFallible(def.parser, ctx, pos);
+  ctx.activeTrivia = savedTrivia;
+  ctx.triviaKindLabels = savedKindLabels;
+  const i = ind(ctx);
+  const sc = v(ctx, "_tokCh");
+  const sl = v(ctx, "_tokLv");
+  const sr = v(ctx, "_tokRaw");
+  const sv = v(ctx, "_tokTr");
+  const sk = v(ctx, "_tokKinds");
+  const stl = v(ctx, "_tokTl");
+  const sol = v(ctx, "_tokLog");
+  const sb = v(ctx, "_tokBuf");
+  const valV = v(ctx, "_tok");
+  return {
+    stmts: [
+      `${i}const ${sc} = _ctx._cstChildren, ${sl} = _ctx._cstLeaves, ${sr} = _ctx._cstRawChildren, ${sv} = _ctx.trivia, ${sk} = _ctx.triviaKindLabels, ${stl} = _ctx._cstTriviaLog, ${sol} = _ctx._triviaLog, ${sb} = _ctx._cstBuf`,
+      `${i}_ctx.trivia = undefined; _ctx.triviaKindLabels = undefined; _ctx._cstChildren = undefined; _ctx._cstLeaves = undefined; _ctx._cstRawChildren = undefined; _ctx._cstTriviaLog = undefined; _ctx._triviaLog = undefined; _ctx._cstBuf = undefined`,
+      `${i}try {`,
+      ...reindentStmts(inner.stmts, ctx.indent + 1).map((stmt) => stmt.replace(/^(\s*)(?:const|let)\s+/, "$1var ")),
+      `${i}} finally {`,
+      `${i}  _ctx.trivia = ${sv}; _ctx.triviaKindLabels = ${sk}; _ctx._cstChildren = ${sc}; _ctx._cstLeaves = ${sl}; _ctx._cstRawChildren = ${sr}; _ctx._cstTriviaLog = ${stl}; _ctx._triviaLog = ${sol}; _ctx._cstBuf = ${sb}`,
+      `${i}}`,
+      ...emitIfFail(ctx, `!${inner.okVar}`, propagateFailBody(ctx)),
+      `${i}const ${valV} = input.slice(${pos}, ${inner.endVar})`,
+      ...emitLeafCapture(ctx, valV, pos, inner.endVar)
+    ],
+    valueVar: valV,
+    endVar: inner.endVar
+  };
+}
 function emitNode(def, ctx, pos) {
   const structural = def.build === void 0;
   const mkType = structural ? null : analyzeMkInlineBuild(def);
@@ -4208,7 +4376,7 @@ function emitRecover(def, ctx, pos) {
     `${ind0}    if (${sentOk}) break`,
     `${ind0}    ${scanV}++`,
     `${ind0}  }`,
-    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(staticExpected(def.parser))} }`,
+    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(deriveExpected(def.parser))} }`,
     `${ind0}  if (_ctx._errors) _ctx._errors.push(${errV})`,
     `${ind0}  ${valVar} = ${errV}`,
     `${ind0}  ${endVar} = ${scanV}`,
@@ -4338,6 +4506,8 @@ function emitDispatch(p, ctx, pos) {
       return emitLazy(p, def, ctx, pos);
     case "trivia":
       return emit(def.parser, ctx, pos);
+    case "token":
+      return emitToken(def, ctx, pos);
     case "label":
       return emit(def.parser, ctx, pos);
     case "grammar": {
@@ -4439,6 +4609,7 @@ function hasNodeDef(p, seen = /* @__PURE__ */ new Set()) {
     }
     case "grammar":
     case "trivia":
+    case "token":
     case "label":
     case "optional":
     case "many":
@@ -4474,6 +4645,7 @@ function childrenOf(def) {
     case "optional":
     case "transform":
     case "trivia":
+    case "token":
     case "label":
     case "grammar":
     case "not":
@@ -4984,7 +5156,7 @@ function balanced(open, close, options = {}) {
   const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
   const inner = many(choice(self, ...skips, content));
   const combi = transform(
-    sequence(literal(open), inner, expect(literal(close), close)),
+    sequence(literal(open), inner, expect(literal(close))),
     // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
     // ParseError (recovered close). `c` is the close string or a ParseError.
     ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
@@ -5046,6 +5218,7 @@ function evalRuleMapIR(ir) {
     "node",
     "parser",
     "scanTo",
+    "token",
     "transform",
     "skip",
     "trivia",
@@ -5071,6 +5244,7 @@ function evalRuleMapIR(ir) {
     node,
     parser,
     scanTo,
+    token,
     transform,
     skip,
     trivia,
@@ -5231,7 +5405,7 @@ function recover(combinator, sentinel) {
     canMatchNewline: true,
     isTrivia: false
   };
-  const expected = staticExpected(combinator);
+  const expected = deriveExpected(combinator);
   return {
     _tag: "recover",
     _meta: meta,
@@ -5944,7 +6118,7 @@ export {
   sequence,
   shiftAbsolute,
   skip,
-  staticExpected,
+  token,
   transform,
   trivia,
   triviaEntries,