parseman 0.17.0 → 0.18.0

package/README.md

@@ -71,7 +71,7 @@ See **[The three modes](https://matthew-dean.github.io/parseman/guide/modes)** f
 
 ## What's in the box
 
-- **[Combinators](https://matthew-dean.github.io/parseman/guide/combinators)** — `literal`, `regex`, `sequence`, `choice`, `many`, `sepBy`, `not`, and more.
+- **[Combinators](https://matthew-dean.github.io/parseman/guide/combinators)** — `literal`, `regex`, `sequence`, `choice`, `many`, `sepBy`, `token`, `not`, and more.
 - **[Whitespace & trivia](https://matthew-dean.github.io/parseman/guide/trivia)** — grammar-defined filler skipping, with per-chunk kind capture.
 - **[Recursive rules](https://matthew-dean.github.io/parseman/guide/recursive-rules)** — `rules()` for mutually recursive grammars; fully macro-compilable.
 - **[CST / AST nodes](https://matthew-dean.github.io/parseman/guide/ast)** — `node()` captures terminals and trivia for you, with a `collapse` option for transparent wrapper rules.

package/dist/combinators/expect.d.ts

@@ -14,7 +14,7 @@ export type { ParseError };
  * the interpreter and the compiled output (the compiled path does not rebuild the
  * runtime `expected` array, so it reads this precomputed set instead).
  */
-export declare function staticExpected(c: Combinator<unknown>): string[];
+export declare function deriveExpected(c: Combinator<unknown>): string[];
 /**
  * Required-token combinator. Tries `combinator`; on success returns it verbatim.
  * On failure it does NOT fail — it records a {@link ParseError} (the statically
@@ -26,7 +26,7 @@ export declare function staticExpected(c: Combinator<unknown>): string[];
  * block) so a missing one is reported with position + expectation rather than
  * aborting the whole parse or being silently swallowed by a catch-all.
  *
- *   sequence(literal('{'), declList, expect(literal('}'), '}'))
+ *   sequence(literal('{'), declList, expect(literal('}')))
  *
  * Errors are only collected when the parse runs with `{ recover: true }`; without
  * it, expect() still recovers in place but records nothing (zero overhead beyond

package/dist/combinators/expect.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../src/combinators/expect.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAyC,UAAU,EAAE,MAAM,aAAa,CAAA;AAEhG,YAAY,EAAE,UAAU,EAAE,CAAA;AAE1B;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,MAAM,EAAE,CA2B/D;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC,CAAC,GAAG,UAAU,CAAC,CAmB/F"}
+{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../src/combinators/expect.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAyC,UAAU,EAAE,MAAM,aAAa,CAAA;AAEhG,YAAY,EAAE,UAAU,EAAE,CAAA;AAE1B;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,MAAM,EAAE,CA4B/D;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC,CAAC,GAAG,UAAU,CAAC,CAmB/F"}

package/dist/combinators/first-set.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"first-set.d.ts","sourceRoot":"","sources":["../../src/combinators/first-set.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAa,QAAQ,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAE7E,wBAAgB,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAKxD;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,GAAG,OAAO,CAS5D;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAE/C;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,QAAQ,CAE1D;AAED,wBAAgB,GAAG,IAAI,QAAQ,CAE9B;AAED,wBAAgB,KAAK,IAAI,QAAQ,CAEhC;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,GAAE,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAa,GAAG,OAAO,CAmCxG;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,QAAQ,CAOlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,GAAE,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAa,GAAG,QAAQ,CAoCvG"}
+{"version":3,"file":"first-set.d.ts","sourceRoot":"","sources":["../../src/combinators/first-set.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAa,QAAQ,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAE7E,wBAAgB,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAKxD;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,GAAG,OAAO,CAS5D;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAE/C;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,QAAQ,CAE1D;AAED,wBAAgB,GAAG,IAAI,QAAQ,CAE9B;AAED,wBAAgB,KAAK,IAAI,QAAQ,CAEhC;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,GAAE,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAa,GAAG,OAAO,CAoCxG;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,QAAQ,CAOlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,GAAE,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAa,GAAG,QAAQ,CAqCvG"}

package/dist/combinators/token.d.ts

@@ -0,0 +1,10 @@
+import type { Combinator } from '../types.ts';
+/**
+ * Treat a parser as one source token.
+ *
+ * The wrapped parser runs with trivia cleared, so its parts must be contiguous.
+ * On success the token returns the matched source text and captures one CST leaf
+ * for the full span, suppressing any internal terminal leaves.
+ */
+export declare function token(root: Combinator<unknown>): Combinator<string>;
+//# sourceMappingURL=token.d.ts.map

package/dist/combinators/token.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../src/combinators/token.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAyC,MAAM,aAAa,CAAA;AAGpF;;;;;;GAMG;AACH,wBAAgB,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAqDnE"}

package/dist/compiler/codegen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"codegen.d.ts","sourceRoot":"","sources":["../../src/compiler/codegen.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,UAAU,EAAa,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAA;AAyBzH;;;;;;;;GAQG;AACH,eAAO,MAAM,eAAe,uTAC0R,CAAA;AActT,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,IAAI,MAAM,EAAE,CAI7C;AA6bD,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,GAAG,MAAM,CAQlE;AAcD,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CAA0B;AAypDjF,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI;IAC9B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAClD,kGAAkG;IAClG,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAChF;;;;OAIG;IACH,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE,CAAA;IACvF,iEAAiE;IACjE,MAAM,EAAE,MAAM,CAAA;IACd;;;;;;OAMG;IACH,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC,CAAA;AAoOD;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAC7D,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA+BvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,CA4GhG;AAgDD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAC7D;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA4FhD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC7B;;;;OAIG;IACH,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IAChC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3B,YAAY,EAAE,OAAO,CAAA;IACrB,cAAc,EAAE,OAAO,CAAA;IACvB;;;;;OAKG;IACH,KAAK,EAAE,aAAa,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,CAAA;IACrD,QAAQ,EAAE,aAAa,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,CAAA;CACzD,CAAA;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EACjE,EAAE,EAAE,MAAM,GACT,cAAc,GAAG,IAAI,CAyKvB"}
+{"version":3,"file":"codegen.d.ts","sourceRoot":"","sources":["../../src/compiler/codegen.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,UAAU,EAAa,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAA;AAyBzH;;;;;;;;GAQG;AACH,eAAO,MAAM,eAAe,uTAC0R,CAAA;AActT,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,IAAI,MAAM,EAAE,CAI7C;AA6bD,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,GAAG,MAAM,CAQlE;AAcD,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CAA0B;AAmxDjF,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI;IAC9B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAClD,kGAAkG;IAClG,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAChF;;;;OAIG;IACH,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG;QAAE,MAAM,EAAE,UAAU,EAAE,CAAA;KAAE,CAAA;IACvF,iEAAiE;IACjE,MAAM,EAAE,MAAM,CAAA;IACd;;;;;;OAMG;IACH,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC,CAAA;AAsOD;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAC7D,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA+BvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,CA4GhG;AAgDD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAC7D;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA4FhD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC7B;;;;OAIG;IACH,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IAChC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3B,YAAY,EAAE,OAAO,CAAA;IACrB,cAAc,EAAE,OAAO,CAAA;IACvB;;;;;OAKG;IACH,KAAK,EAAE,aAAa,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,CAAA;IACrD,QAAQ,EAAE,aAAa,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,CAAA;CACzD,CAAA;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EACjE,EAAE,EAAE,MAAM,GACT,cAAc,GAAG,IAAI,CAyKvB"}

package/dist/compiler/ir-serialize.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ir-serialize.d.ts","sourceRoot":"","sources":["../../src/compiler/ir-serialize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,KAAK,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAgBxD,KAAK,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;AA0C/B,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,GAC9C,MAAM,GAAG,IAAI,CAUf;AAED;;oFAEoF;AACpF,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAgC/D"}
+{"version":3,"file":"ir-serialize.d.ts","sourceRoot":"","sources":["../../src/compiler/ir-serialize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,KAAK,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAiBxD,KAAK,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;AA2C/B,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,GAC9C,MAAM,GAAG,IAAI,CAUf;AAED;;oFAEoF;AACpF,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAgC/D"}

package/dist/compiler/value-usage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"value-usage.d.ts","sourceRoot":"","sources":["../../src/compiler/value-usage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAExD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,CAuGhE"}
+{"version":3,"file":"value-usage.d.ts","sourceRoot":"","sources":["../../src/compiler/value-usage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAExD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,CAwGhE"}

package/dist/index.cjs

@@ -85,7 +85,7 @@ __export(index_exports, {
   sequence: () => sequence,
   shiftAbsolute: () => shiftAbsolute,
   skip: () => skip,
-  staticExpected: () => staticExpected,
+  token: () => token,
   transform: () => transform,
   trivia: () => trivia,
   triviaEntries: () => triviaEntries,
@@ -155,6 +155,7 @@ function matchesEmpty(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "expect":
     case "withCtx":
     case "node":
@@ -217,6 +218,7 @@ function firstSetOf(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "node":
     case "grammar":
     case "expect":
@@ -1343,6 +1345,7 @@ function markUnusedValues(root) {
         return;
       case "label":
       case "trivia":
+      case "token":
       case "withCtx":
       case "expect":
         visit(def.parser, consumed);
@@ -1643,8 +1646,60 @@ function parse(combinator, input, opts = {}) {
   return withErrors;
 }
 
+// src/combinators/token.ts
+function token(root) {
+  const meta = {
+    firstSet: root._meta.firstSet,
+    canMatchNewline: root._meta.canMatchNewline,
+    isTrivia: false
+  };
+  return {
+    _tag: "token",
+    _meta: meta,
+    _def: { tag: "token", parser: root },
+    parse(input, pos, ctx) {
+      const savedTrivia = ctx.trivia;
+      const savedKinds = ctx.triviaKindLabels;
+      const savedBuf = ctx._cstBuf;
+      const savedChildren = ctx._cstChildren;
+      const savedLeaves = ctx._cstLeaves;
+      const savedRaw = ctx._cstRawChildren;
+      const savedTriviaLog = ctx._cstTriviaLog;
+      const savedOuterTriviaLog = ctx._triviaLog;
+      const wasCapturing = cstCaptureActive(ctx);
+      ctx.trivia = void 0;
+      ctx.triviaKindLabels = void 0;
+      ctx._cstBuf = void 0;
+      ctx._cstChildren = void 0;
+      ctx._cstLeaves = void 0;
+      ctx._cstRawChildren = void 0;
+      ctx._cstTriviaLog = void 0;
+      delete ctx._triviaLog;
+      let result;
+      try {
+        result = root.parse(input, pos, ctx);
+      } finally {
+        ctx.trivia = savedTrivia;
+        ctx.triviaKindLabels = savedKinds;
+        ctx._cstBuf = savedBuf;
+        ctx._cstChildren = savedChildren;
+        ctx._cstLeaves = savedLeaves;
+        ctx._cstRawChildren = savedRaw;
+        ctx._cstTriviaLog = savedTriviaLog;
+        if (savedOuterTriviaLog === void 0) delete ctx._triviaLog;
+        else ctx._triviaLog = savedOuterTriviaLog;
+      }
+      if (!result.ok) return result;
+      const span = result.span;
+      const value = input.slice(pos, span.end);
+      if (wasCapturing) pushCstLeaf(ctx, { _tag: "leaf", value, span });
+      return { ok: true, value, span };
+    }
+  };
+}
+
 // src/combinators/expect.ts
-function staticExpected(c) {
+function deriveExpected(c) {
   const def = c._def;
   switch (def.tag) {
     case "literal":
@@ -1656,22 +1711,23 @@ function staticExpected(c) {
     case "label":
       return [def.label];
     case "choice":
-      return def.parsers.flatMap(staticExpected);
+      return def.parsers.flatMap(deriveExpected);
     case "sequence":
-      return def.parsers.length > 0 ? staticExpected(def.parsers[0]) : [];
+      return def.parsers.length > 0 ? deriveExpected(def.parsers[0]) : [];
     case "node":
     case "grammar":
     case "trivia":
+    case "token":
     case "optional":
     case "many":
     case "oneOrMore":
     case "transform":
     case "not":
-      return staticExpected(def.parser);
+      return deriveExpected(def.parser);
     case "lazy": {
       const name = c._ruleName;
       try {
-        return staticExpected(def.thunk());
+        return deriveExpected(def.thunk());
       } catch {
         return name ? [name] : [];
       }
@@ -1681,7 +1737,7 @@ function staticExpected(c) {
   }
 }
 function expect(combinator, label2) {
-  const expected = label2 !== void 0 ? [label2] : staticExpected(combinator);
+  const expected = label2 !== void 0 ? [label2] : deriveExpected(combinator);
   const meta = {
     firstSet: combinator._meta.firstSet,
     canMatchNewline: combinator._meta.canMatchNewline,
@@ -3333,6 +3389,7 @@ function mayLeavePartialCapture(p, seen = /* @__PURE__ */ new Set()) {
     case "guard":
     case "not":
     case "trivia":
+    case "token":
     case "scanTo":
     case "unknown":
       return false;
@@ -3394,6 +3451,7 @@ function capturesLeaf(p, seen = /* @__PURE__ */ new Set()) {
     case "regex":
     case "keywords":
     case "node":
+    case "token":
       return true;
     case "not":
     case "guard":
@@ -3672,9 +3730,9 @@ function emitSeq(def, ctx, pos) {
   stmts.push(`${ind(ctx)}const ${arrV} = [${valueVars.join(", ")}]`);
   return { stmts, valueVar: arrV, endVar };
 }
-function staticExpectedArr(parsers) {
+function deriveExpectedArr(parsers) {
   return JSON.stringify(parsers.flatMap((p) => {
-    const e = staticExpected(p);
+    const e = deriveExpected(p);
     return e.length > 0 ? e : [p._tag];
   }));
 }
@@ -3695,7 +3753,7 @@ function failsAtStart(p) {
   }
 }
 function armStaticExpected(ctx, p) {
-  return hoistExpected(ctx, staticExpectedArr([p]));
+  return hoistExpected(ctx, deriveExpectedArr([p]));
 }
 function canMatchEmptyAtStart(p) {
   const d = p._def;
@@ -3733,7 +3791,7 @@ function emitDisjointArm(p, ctx, pos, valV, endV) {
   ];
 }
 function emitChoice(def, ctx, pos) {
-  const allExpected = staticExpectedArr(def.parsers);
+  const allExpected = deriveExpectedArr(def.parsers);
   if (def.disjoint) {
     const codeV = v(ctx, "_code");
     const valV = v(ctx, "_chv");
@@ -3794,7 +3852,7 @@ function emitGreedyClassify(def, superIndex, ctx, pos) {
   const wordV = v(ctx, "_gw");
   const endV = v(ctx, "_ge");
   const valV = v(ctx, "_gcv");
-  const regexExpected = JSON.stringify(staticExpected(superParser));
+  const regexExpected = JSON.stringify(deriveExpected(superParser));
   const stmts = [
     `${ind(ctx)}${reVar}.lastIndex = ${pos}`,
     `${ind(ctx)}const ${matchV} = ${reVar}.exec(input)`,
@@ -4186,6 +4244,116 @@ function emitNot(def, ctx, pos) {
     endVar: pos
   };
 }
+function escapeTokenLiteral(value) {
+  return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+}
+function regexMatchesEmpty(source, flags) {
+  try {
+    return new RegExp(`^(?:${source})$`, flags).test("");
+  } catch {
+    return true;
+  }
+}
+function tokenTerminalSource(p) {
+  const def = p._def;
+  switch (def.tag) {
+    case "literal":
+      if (def.value.length === 0) return null;
+      return {
+        source: escapeTokenLiteral(def.value),
+        caseInsensitive: def.caseInsensitive,
+        rawCaseSensitive: def.caseInsensitive ? "" : def.value
+      };
+    case "regex": {
+      if (def.flags.replace(/i/g, "") !== "") return null;
+      if (regexMatchesEmpty(def.source, def.flags)) return null;
+      return {
+        source: `(?:${def.source})`,
+        caseInsensitive: def.flags.includes("i"),
+        rawCaseSensitive: def.flags.includes("i") ? "" : def.source
+      };
+    }
+    default:
+      return null;
+  }
+}
+function tokenSequenceSource(p) {
+  const def = p._def;
+  const parts = def.tag === "sequence" ? def.parsers.map(tokenTerminalSource) : [tokenTerminalSource(p)];
+  if (parts.some((part) => part === null)) return null;
+  const typed = parts;
+  const flags = typed.some((part) => part.caseInsensitive) ? "i" : "";
+  if (flags.includes("i") && typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))) return null;
+  return {
+    source: typed.map((part) => part.source).join(""),
+    flags,
+    hasCaseSensitiveLetters: typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))
+  };
+}
+function combineTokenRegexSources(parts) {
+  const flags = parts.some((part) => part.flags.includes("i")) ? "i" : "";
+  if (flags.includes("i") && parts.some((part) => part.flags === "" && part.hasCaseSensitiveLetters)) return null;
+  return { source: parts.map((part) => part.source).join(""), flags };
+}
+function tokenNullableRegex(def) {
+  const inner = def.parser._def;
+  if (inner.tag === "many") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})*`, flags: seq.flags };
+  }
+  if (inner.tag === "optional") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})?`, flags: seq.flags };
+  }
+  if (inner.tag === "sepBy") {
+    const item = tokenSequenceSource(inner.parser);
+    const sep = tokenSequenceSource(inner.separator);
+    if (!item || !sep) return null;
+    const combined = combineTokenRegexSources([item, sep]);
+    if (!combined) return null;
+    return { tag: "regex", source: `(?:${item.source}(?:${sep.source}${item.source})*)?`, flags: combined.flags };
+  }
+  return null;
+}
+function emitToken(def, ctx, pos) {
+  const lowered = tokenNullableRegex(def);
+  if (lowered) return emitRegex(lowered, ctx, pos);
+  const savedTrivia = ctx.activeTrivia;
+  const savedKindLabels = ctx.triviaKindLabels;
+  ctx.activeTrivia = void 0;
+  ctx.triviaKindLabels = void 0;
+  const inner = emitFallible(def.parser, ctx, pos);
+  ctx.activeTrivia = savedTrivia;
+  ctx.triviaKindLabels = savedKindLabels;
+  const i = ind(ctx);
+  const sc = v(ctx, "_tokCh");
+  const sl = v(ctx, "_tokLv");
+  const sr = v(ctx, "_tokRaw");
+  const sv = v(ctx, "_tokTr");
+  const sk = v(ctx, "_tokKinds");
+  const stl = v(ctx, "_tokTl");
+  const sol = v(ctx, "_tokLog");
+  const sb = v(ctx, "_tokBuf");
+  const valV = v(ctx, "_tok");
+  return {
+    stmts: [
+      `${i}const ${sc} = _ctx._cstChildren, ${sl} = _ctx._cstLeaves, ${sr} = _ctx._cstRawChildren, ${sv} = _ctx.trivia, ${sk} = _ctx.triviaKindLabels, ${stl} = _ctx._cstTriviaLog, ${sol} = _ctx._triviaLog, ${sb} = _ctx._cstBuf`,
+      `${i}_ctx.trivia = undefined; _ctx.triviaKindLabels = undefined; _ctx._cstChildren = undefined; _ctx._cstLeaves = undefined; _ctx._cstRawChildren = undefined; _ctx._cstTriviaLog = undefined; _ctx._triviaLog = undefined; _ctx._cstBuf = undefined`,
+      `${i}try {`,
+      ...reindentStmts(inner.stmts, ctx.indent + 1).map((stmt) => stmt.replace(/^(\s*)(?:const|let)\s+/, "$1var ")),
+      `${i}} finally {`,
+      `${i}  _ctx.trivia = ${sv}; _ctx.triviaKindLabels = ${sk}; _ctx._cstChildren = ${sc}; _ctx._cstLeaves = ${sl}; _ctx._cstRawChildren = ${sr}; _ctx._cstTriviaLog = ${stl}; _ctx._triviaLog = ${sol}; _ctx._cstBuf = ${sb}`,
+      `${i}}`,
+      ...emitIfFail(ctx, `!${inner.okVar}`, propagateFailBody(ctx)),
+      `${i}const ${valV} = input.slice(${pos}, ${inner.endVar})`,
+      ...emitLeafCapture(ctx, valV, pos, inner.endVar)
+    ],
+    valueVar: valV,
+    endVar: inner.endVar
+  };
+}
 function emitNode(def, ctx, pos) {
   const structural = def.build === void 0;
   const mkType = structural ? null : analyzeMkInlineBuild(def);
@@ -4305,7 +4473,7 @@ function emitRecover(def, ctx, pos) {
     `${ind0}    if (${sentOk}) break`,
     `${ind0}    ${scanV}++`,
     `${ind0}  }`,
-    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(staticExpected(def.parser))} }`,
+    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(deriveExpected(def.parser))} }`,
     `${ind0}  if (_ctx._errors) _ctx._errors.push(${errV})`,
     `${ind0}  ${valVar} = ${errV}`,
     `${ind0}  ${endVar} = ${scanV}`,
@@ -4435,6 +4603,8 @@ function emitDispatch(p, ctx, pos) {
       return emitLazy(p, def, ctx, pos);
     case "trivia":
       return emit(def.parser, ctx, pos);
+    case "token":
+      return emitToken(def, ctx, pos);
     case "label":
       return emit(def.parser, ctx, pos);
     case "grammar": {
@@ -4536,6 +4706,7 @@ function hasNodeDef(p, seen = /* @__PURE__ */ new Set()) {
     }
     case "grammar":
     case "trivia":
+    case "token":
     case "label":
     case "optional":
     case "many":
@@ -4571,6 +4742,7 @@ function childrenOf(def) {
     case "optional":
     case "transform":
     case "trivia":
+    case "token":
     case "label":
     case "grammar":
     case "not":
@@ -5081,7 +5253,7 @@ function balanced(open, close, options = {}) {
   const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
   const inner = many(choice(self, ...skips, content));
   const combi = transform(
-    sequence(literal(open), inner, expect(literal(close), close)),
+    sequence(literal(open), inner, expect(literal(close))),
     // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
     // ParseError (recovered close). `c` is the close string or a ParseError.
     ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
@@ -5143,6 +5315,7 @@ function evalRuleMapIR(ir) {
     "node",
     "parser",
     "scanTo",
+    "token",
     "transform",
     "skip",
     "trivia",
@@ -5168,6 +5341,7 @@ function evalRuleMapIR(ir) {
     node,
     parser,
     scanTo,
+    token,
     transform,
     skip,
     trivia,
@@ -5328,7 +5502,7 @@ function recover(combinator, sentinel) {
     canMatchNewline: true,
     isTrivia: false
   };
-  const expected = staticExpected(combinator);
+  const expected = deriveExpected(combinator);
   return {
     _tag: "recover",
     _meta: meta,