parseman 0.16.0 → 0.18.0

package/dist/index.cjs

@@ -85,7 +85,7 @@ __export(index_exports, {
   sequence: () => sequence,
   shiftAbsolute: () => shiftAbsolute,
   skip: () => skip,
-  staticExpected: () => staticExpected,
+  token: () => token,
   transform: () => transform,
   trivia: () => trivia,
   triviaEntries: () => triviaEntries,
@@ -155,6 +155,7 @@ function matchesEmpty(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "expect":
     case "withCtx":
     case "node":
@@ -181,6 +182,55 @@ function sequenceFirstSet(parsers) {
   }
   return fs;
 }
+function firstSetOf(p, seen = /* @__PURE__ */ new Set()) {
+  if (seen.has(p)) return any();
+  seen.add(p);
+  const fs = (c) => firstSetOf(c, seen);
+  const d = p._def;
+  switch (d.tag) {
+    case "literal":
+    case "regex":
+    case "keywords":
+      return p._meta.firstSet;
+    // terminals: no refs, cached set is exact
+    case "lazy":
+      try {
+        return fs(d.thunk());
+      } catch {
+        return any();
+      }
+    case "choice": {
+      let out = empty();
+      for (const arm of d.parsers) out = union(out, fs(arm));
+      return out;
+    }
+    case "sequence": {
+      let out = empty();
+      for (const term of d.parsers) {
+        out = union(out, fs(term));
+        if (!matchesEmpty(term)) return out;
+      }
+      return out;
+    }
+    case "oneOrMore":
+    case "many":
+    case "optional":
+    case "transform":
+    case "label":
+    case "trivia":
+    case "token":
+    case "node":
+    case "grammar":
+    case "expect":
+      return fs(d.parser);
+    case "sepBy":
+      return fs(d.parser);
+    case "skip":
+      return fs(d.main);
+    default:
+      return p._meta.firstSet;
+  }
+}
 function mergeRanges(ranges) {
   if (ranges.length === 0) return [];
   const sorted = [...ranges].sort((a, b) => a.lo - b.lo);
@@ -1295,6 +1345,7 @@ function markUnusedValues(root) {
         return;
       case "label":
       case "trivia":
+      case "token":
       case "withCtx":
       case "expect":
         visit(def.parser, consumed);
@@ -1595,8 +1646,60 @@ function parse(combinator, input, opts = {}) {
   return withErrors;
 }
 
+// src/combinators/token.ts
+function token(root) {
+  const meta = {
+    firstSet: root._meta.firstSet,
+    canMatchNewline: root._meta.canMatchNewline,
+    isTrivia: false
+  };
+  return {
+    _tag: "token",
+    _meta: meta,
+    _def: { tag: "token", parser: root },
+    parse(input, pos, ctx) {
+      const savedTrivia = ctx.trivia;
+      const savedKinds = ctx.triviaKindLabels;
+      const savedBuf = ctx._cstBuf;
+      const savedChildren = ctx._cstChildren;
+      const savedLeaves = ctx._cstLeaves;
+      const savedRaw = ctx._cstRawChildren;
+      const savedTriviaLog = ctx._cstTriviaLog;
+      const savedOuterTriviaLog = ctx._triviaLog;
+      const wasCapturing = cstCaptureActive(ctx);
+      ctx.trivia = void 0;
+      ctx.triviaKindLabels = void 0;
+      ctx._cstBuf = void 0;
+      ctx._cstChildren = void 0;
+      ctx._cstLeaves = void 0;
+      ctx._cstRawChildren = void 0;
+      ctx._cstTriviaLog = void 0;
+      delete ctx._triviaLog;
+      let result;
+      try {
+        result = root.parse(input, pos, ctx);
+      } finally {
+        ctx.trivia = savedTrivia;
+        ctx.triviaKindLabels = savedKinds;
+        ctx._cstBuf = savedBuf;
+        ctx._cstChildren = savedChildren;
+        ctx._cstLeaves = savedLeaves;
+        ctx._cstRawChildren = savedRaw;
+        ctx._cstTriviaLog = savedTriviaLog;
+        if (savedOuterTriviaLog === void 0) delete ctx._triviaLog;
+        else ctx._triviaLog = savedOuterTriviaLog;
+      }
+      if (!result.ok) return result;
+      const span = result.span;
+      const value = input.slice(pos, span.end);
+      if (wasCapturing) pushCstLeaf(ctx, { _tag: "leaf", value, span });
+      return { ok: true, value, span };
+    }
+  };
+}
+
 // src/combinators/expect.ts
-function staticExpected(c) {
+function deriveExpected(c) {
   const def = c._def;
   switch (def.tag) {
     case "literal":
@@ -1608,22 +1711,23 @@ function staticExpected(c) {
     case "label":
       return [def.label];
     case "choice":
-      return def.parsers.flatMap(staticExpected);
+      return def.parsers.flatMap(deriveExpected);
     case "sequence":
-      return def.parsers.length > 0 ? staticExpected(def.parsers[0]) : [];
+      return def.parsers.length > 0 ? deriveExpected(def.parsers[0]) : [];
     case "node":
     case "grammar":
     case "trivia":
+    case "token":
     case "optional":
     case "many":
     case "oneOrMore":
     case "transform":
     case "not":
-      return staticExpected(def.parser);
+      return deriveExpected(def.parser);
     case "lazy": {
       const name = c._ruleName;
       try {
-        return staticExpected(def.thunk());
+        return deriveExpected(def.thunk());
       } catch {
         return name ? [name] : [];
       }
@@ -1633,7 +1737,7 @@ function staticExpected(c) {
   }
 }
 function expect(combinator, label2) {
-  const expected = label2 !== void 0 ? [label2] : staticExpected(combinator);
+  const expected = label2 !== void 0 ? [label2] : deriveExpected(combinator);
   const meta = {
     firstSet: combinator._meta.firstSet,
     canMatchNewline: combinator._meta.canMatchNewline,
@@ -3121,16 +3225,12 @@ function namedFnPrelude() {
 function pushNamedFnDecl(ctx, fnName, bodyStmts, valueVar, endVar) {
   ctx.namedFnDecls.push([
     `function ${fnName}(input, _pos, _ctx) {`,
-    `  let _pfok = false, _pfv, _pfe = _pos`,
     `  _pfail: {`,
     ...reindentStmts(bodyStmts, 2),
-    `    _pfv = ${valueVar}`,
-    `    _pfe = ${endVar}`,
-    `    _pfok = true`,
+    `    ${NAMED_FN_END} = ${endVar}`,
+    `    return ${valueVar}`,
     `  }`,
-    `  if (!_pfok) return ${NAMED_FN_FAIL}`,
-    `  ${NAMED_FN_END} = _pfe`,
-    `  return _pfv`,
+    `  return ${NAMED_FN_FAIL}`,
     `}`
   ].join("\n"));
 }
@@ -3289,6 +3389,7 @@ function mayLeavePartialCapture(p, seen = /* @__PURE__ */ new Set()) {
     case "guard":
     case "not":
     case "trivia":
+    case "token":
     case "scanTo":
     case "unknown":
       return false;
@@ -3350,6 +3451,7 @@ function capturesLeaf(p, seen = /* @__PURE__ */ new Set()) {
     case "regex":
     case "keywords":
     case "node":
+    case "token":
       return true;
     case "not":
     case "guard":
@@ -3628,9 +3730,9 @@ function emitSeq(def, ctx, pos) {
   stmts.push(`${ind(ctx)}const ${arrV} = [${valueVars.join(", ")}]`);
   return { stmts, valueVar: arrV, endVar };
 }
-function staticExpectedArr(parsers) {
+function deriveExpectedArr(parsers) {
   return JSON.stringify(parsers.flatMap((p) => {
-    const e = staticExpected(p);
+    const e = deriveExpected(p);
     return e.length > 0 ? e : [p._tag];
   }));
 }
@@ -3651,7 +3753,7 @@ function failsAtStart(p) {
   }
 }
 function armStaticExpected(ctx, p) {
-  return hoistExpected(ctx, staticExpectedArr([p]));
+  return hoistExpected(ctx, deriveExpectedArr([p]));
 }
 function canMatchEmptyAtStart(p) {
   const d = p._def;
@@ -3689,7 +3791,7 @@ function emitDisjointArm(p, ctx, pos, valV, endV) {
   ];
 }
 function emitChoice(def, ctx, pos) {
-  const allExpected = staticExpectedArr(def.parsers);
+  const allExpected = deriveExpectedArr(def.parsers);
   if (def.disjoint) {
     const codeV = v(ctx, "_code");
     const valV = v(ctx, "_chv");
@@ -3750,7 +3852,7 @@ function emitGreedyClassify(def, superIndex, ctx, pos) {
   const wordV = v(ctx, "_gw");
   const endV = v(ctx, "_ge");
   const valV = v(ctx, "_gcv");
-  const regexExpected = JSON.stringify(staticExpected(superParser));
+  const regexExpected = JSON.stringify(deriveExpected(superParser));
   const stmts = [
     `${ind(ctx)}${reVar}.lastIndex = ${pos}`,
     `${ind(ctx)}const ${matchV} = ${reVar}.exec(input)`,
@@ -3831,11 +3933,16 @@ function emitFirstMatch(def, ctx, pos) {
     const atStart = failsAtStart(p);
     const staticFx = armStaticExpected(ctx, p);
     const deferRuleName = ctx.deferFirstSetRefs ? p._ruleName : void 0;
-    const fsGuard = deferRuleName !== void 0 ? `/*@FS:${deferRuleName}:${codeV}@*/true` : needsFirstSetGuard(p) ? firstSetCond(codeV, p._meta.firstSet) : null;
+    const fsGuard = deferRuleName !== void 0 ? `/*@FS:${deferRuleName}:${codeV}@*/true` : needsFirstSetGuard(p) ? firstSetCond(codeV, p._meta.firstSet) : (() => {
+      if (p._meta.firstSet.kind !== "any" || matchesEmpty(p)) return null;
+      const deep = firstSetOf(p);
+      return deep.kind === "ranges" ? firstSetCond(codeV, deep) : null;
+    })();
     let gateCond = null;
     if (gate) {
       const gateIdx = ctx.mapFns.length;
       ctx.mapFns.push(gate);
+      ctx.mapFnSrcs.push(null);
       gateCond = `${mfRef(ctx)}[${gateIdx}](_ctx.state)`;
     }
     const skipCond = gateCond ? `!${resOkV} && ${gateCond}` : `!${resOkV}`;
@@ -3894,13 +4001,21 @@ function emitNonDisjoint(def, strategy, allExpected, ctx, pos) {
     return emitLiteralsLongestFirst(def, strategy.sortedIndices, allExpected, ctx, pos);
   return emitFirstMatch(def, ctx, pos);
 }
+function pushMapFn(ctx, fn, src) {
+  if (src !== null && ctx.mapFns.length === ctx.mapFnSrcs.length) {
+    const hit = ctx.mapFnSrcs.indexOf(src);
+    if (hit !== -1) return hit;
+  }
+  const idx = ctx.mapFns.length;
+  ctx.mapFns.push(fn);
+  ctx.mapFnSrcs.push(src);
+  return idx;
+}
 function emitTransformChain(p, baseValue, endV, startPos, ctx) {
   const def = p._def;
   if (def.tag === "transform") {
     const innerR = emitTransformChain(def.parser, baseValue, endV, startPos, ctx);
-    const fnIdx = ctx.mapFns.length;
-    ctx.mapFns.push(def.fn);
-    ctx.mapFnSrcs.push(def.fnSrc ?? null);
+    const fnIdx = pushMapFn(ctx, def.fn, def.fnSrc ?? null);
     const vv = v(ctx);
     return {
       stmts: [...innerR.stmts, `${ind(ctx)}const ${vv} = ${mfRef(ctx)}[${fnIdx}](${innerR.valueVar}, { start: ${startPos}, end: ${endV} })`],
@@ -4083,7 +4198,7 @@ function emitScanTo(def, ctx, pos) {
     `${ind(ctx)}while (${curV} < input.length) {`
   ];
   ctx.indent++;
-  const probeCtx = { ...ctx, capturing: false };
+  const probeCtx = { ...ctx, capturing: false, noHoist: true };
   const { stmts: sentStmts, okVar: sentOk } = emitFallible(def.sentinel, probeCtx, curV);
   stmts.push(...sentStmts, `${ind(ctx)}if (${sentOk}) { ${foundV} = true; break }`);
   if (def.skip.length > 0) {
@@ -4129,6 +4244,116 @@ function emitNot(def, ctx, pos) {
     endVar: pos
   };
 }
+function escapeTokenLiteral(value) {
+  return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+}
+function regexMatchesEmpty(source, flags) {
+  try {
+    return new RegExp(`^(?:${source})$`, flags).test("");
+  } catch {
+    return true;
+  }
+}
+function tokenTerminalSource(p) {
+  const def = p._def;
+  switch (def.tag) {
+    case "literal":
+      if (def.value.length === 0) return null;
+      return {
+        source: escapeTokenLiteral(def.value),
+        caseInsensitive: def.caseInsensitive,
+        rawCaseSensitive: def.caseInsensitive ? "" : def.value
+      };
+    case "regex": {
+      if (def.flags.replace(/i/g, "") !== "") return null;
+      if (regexMatchesEmpty(def.source, def.flags)) return null;
+      return {
+        source: `(?:${def.source})`,
+        caseInsensitive: def.flags.includes("i"),
+        rawCaseSensitive: def.flags.includes("i") ? "" : def.source
+      };
+    }
+    default:
+      return null;
+  }
+}
+function tokenSequenceSource(p) {
+  const def = p._def;
+  const parts = def.tag === "sequence" ? def.parsers.map(tokenTerminalSource) : [tokenTerminalSource(p)];
+  if (parts.some((part) => part === null)) return null;
+  const typed = parts;
+  const flags = typed.some((part) => part.caseInsensitive) ? "i" : "";
+  if (flags.includes("i") && typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))) return null;
+  return {
+    source: typed.map((part) => part.source).join(""),
+    flags,
+    hasCaseSensitiveLetters: typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))
+  };
+}
+function combineTokenRegexSources(parts) {
+  const flags = parts.some((part) => part.flags.includes("i")) ? "i" : "";
+  if (flags.includes("i") && parts.some((part) => part.flags === "" && part.hasCaseSensitiveLetters)) return null;
+  return { source: parts.map((part) => part.source).join(""), flags };
+}
+function tokenNullableRegex(def) {
+  const inner = def.parser._def;
+  if (inner.tag === "many") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})*`, flags: seq.flags };
+  }
+  if (inner.tag === "optional") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})?`, flags: seq.flags };
+  }
+  if (inner.tag === "sepBy") {
+    const item = tokenSequenceSource(inner.parser);
+    const sep = tokenSequenceSource(inner.separator);
+    if (!item || !sep) return null;
+    const combined = combineTokenRegexSources([item, sep]);
+    if (!combined) return null;
+    return { tag: "regex", source: `(?:${item.source}(?:${sep.source}${item.source})*)?`, flags: combined.flags };
+  }
+  return null;
+}
+function emitToken(def, ctx, pos) {
+  const lowered = tokenNullableRegex(def);
+  if (lowered) return emitRegex(lowered, ctx, pos);
+  const savedTrivia = ctx.activeTrivia;
+  const savedKindLabels = ctx.triviaKindLabels;
+  ctx.activeTrivia = void 0;
+  ctx.triviaKindLabels = void 0;
+  const inner = emitFallible(def.parser, ctx, pos);
+  ctx.activeTrivia = savedTrivia;
+  ctx.triviaKindLabels = savedKindLabels;
+  const i = ind(ctx);
+  const sc = v(ctx, "_tokCh");
+  const sl = v(ctx, "_tokLv");
+  const sr = v(ctx, "_tokRaw");
+  const sv = v(ctx, "_tokTr");
+  const sk = v(ctx, "_tokKinds");
+  const stl = v(ctx, "_tokTl");
+  const sol = v(ctx, "_tokLog");
+  const sb = v(ctx, "_tokBuf");
+  const valV = v(ctx, "_tok");
+  return {
+    stmts: [
+      `${i}const ${sc} = _ctx._cstChildren, ${sl} = _ctx._cstLeaves, ${sr} = _ctx._cstRawChildren, ${sv} = _ctx.trivia, ${sk} = _ctx.triviaKindLabels, ${stl} = _ctx._cstTriviaLog, ${sol} = _ctx._triviaLog, ${sb} = _ctx._cstBuf`,
+      `${i}_ctx.trivia = undefined; _ctx.triviaKindLabels = undefined; _ctx._cstChildren = undefined; _ctx._cstLeaves = undefined; _ctx._cstRawChildren = undefined; _ctx._cstTriviaLog = undefined; _ctx._triviaLog = undefined; _ctx._cstBuf = undefined`,
+      `${i}try {`,
+      ...reindentStmts(inner.stmts, ctx.indent + 1).map((stmt) => stmt.replace(/^(\s*)(?:const|let)\s+/, "$1var ")),
+      `${i}} finally {`,
+      `${i}  _ctx.trivia = ${sv}; _ctx.triviaKindLabels = ${sk}; _ctx._cstChildren = ${sc}; _ctx._cstLeaves = ${sl}; _ctx._cstRawChildren = ${sr}; _ctx._cstTriviaLog = ${stl}; _ctx._triviaLog = ${sol}; _ctx._cstBuf = ${sb}`,
+      `${i}}`,
+      ...emitIfFail(ctx, `!${inner.okVar}`, propagateFailBody(ctx)),
+      `${i}const ${valV} = input.slice(${pos}, ${inner.endVar})`,
+      ...emitLeafCapture(ctx, valV, pos, inner.endVar)
+    ],
+    valueVar: valV,
+    endVar: inner.endVar
+  };
+}
 function emitNode(def, ctx, pos) {
   const structural = def.build === void 0;
   const mkType = structural ? null : analyzeMkInlineBuild(def);
@@ -4248,7 +4473,7 @@ function emitRecover(def, ctx, pos) {
     `${ind0}    if (${sentOk}) break`,
     `${ind0}    ${scanV}++`,
     `${ind0}  }`,
-    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(staticExpected(def.parser))} }`,
+    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(deriveExpected(def.parser))} }`,
     `${ind0}  if (_ctx._errors) _ctx._errors.push(${errV})`,
     `${ind0}  ${valVar} = ${errV}`,
     `${ind0}  ${endVar} = ${scanV}`,
@@ -4274,6 +4499,32 @@ function emitExpect(def, ctx, pos) {
   return { stmts, valueVar: valVar, endVar };
 }
 function emit(p, ctx, pos) {
+  const usage = ctx.lazyUsage;
+  if (usage && !ctx.noHoist && !ctx.capAsTrivia && !ctx.ruleNames?.has(p) && isHoistableTag(p._def.tag) && (usage.counts.get(p) ?? 0) > 1 && // Only hoist subtrees big enough that de-duplicating them beats the call
+  // overhead. A tiny shared wrapper (e.g. `optional(literal(';'))`) inlines
+  // faster than it calls and barely moves the byte count; the size explosion
+  // is all in LARGE choices/sequences referenced from many positions.
+  (usage.sizes.get(p) ?? 0) >= HOIST_MIN_SUBTREE) {
+    const existing = ctx.namedParsers.get(p);
+    if (existing !== void 0) return emitNamedFnCall(ctx, existing, pos);
+    const fnName = `${nsp(ctx)}_pf${ctx.namedParsers.size}`;
+    ctx.namedParsers.set(p, fnName);
+    const savedIndent = ctx.indent;
+    const savedFailLabel = ctx.failLabel;
+    const savedRecord = ctx.recordFail;
+    ctx.indent = 1;
+    ctx.failLabel = "_pfail";
+    ctx.recordFail = true;
+    const r = emitDispatch(p, ctx, "_pos");
+    ctx.indent = savedIndent;
+    ctx.failLabel = savedFailLabel;
+    ctx.recordFail = savedRecord;
+    pushNamedFnDecl(ctx, fnName, r.stmts, r.valueVar, r.endVar);
+    return emitNamedFnCall(ctx, fnName, pos);
+  }
+  return emitDispatch(p, ctx, pos);
+}
+function emitDispatch(p, ctx, pos) {
   const def = p._def;
   switch (def.tag) {
     case "literal":
@@ -4319,9 +4570,7 @@ function emit(p, ctx, pos) {
           };
         }
       }
-      const fnIdx = ctx.mapFns.length;
-      ctx.mapFns.push(def.fn);
-      ctx.mapFnSrcs.push(def.fnSrc ?? null);
+      const fnIdx = pushMapFn(ctx, def.fn, def.fnSrc ?? null);
       const mv = v(ctx, "_mapped");
       return {
         stmts: [
@@ -4354,6 +4603,8 @@ function emit(p, ctx, pos) {
       return emitLazy(p, def, ctx, pos);
     case "trivia":
       return emit(def.parser, ctx, pos);
+    case "token":
+      return emitToken(def, ctx, pos);
     case "label":
       return emit(def.parser, ctx, pos);
     case "grammar": {
@@ -4386,6 +4637,7 @@ function emit(p, ctx, pos) {
     case "guard": {
       const fnIdx = ctx.mapFns.length;
       ctx.mapFns.push(def.predicate);
+      ctx.mapFnSrcs.push(null);
       const vv = v(ctx);
       return {
         stmts: [
@@ -4400,6 +4652,7 @@ function emit(p, ctx, pos) {
       const evIdx = ctx.mapFns.length;
       const extra = def.extra;
       ctx.mapFns.push((() => extra));
+      ctx.mapFnSrcs.push(null);
       const innerParser = def.parser;
       if (!ctx.namedParsers.has(innerParser)) {
         const fnName = `_wcf${ctx.namedParsers.size}`;
@@ -4453,6 +4706,7 @@ function hasNodeDef(p, seen = /* @__PURE__ */ new Set()) {
     }
     case "grammar":
     case "trivia":
+    case "token":
     case "label":
     case "optional":
     case "many":
@@ -4488,6 +4742,7 @@ function childrenOf(def) {
     case "optional":
     case "transform":
     case "trivia":
+    case "token":
     case "label":
     case "grammar":
     case "not":
@@ -4519,9 +4774,58 @@ function childrenOf(def) {
       return [];
   }
 }
+var HOISTABLE_TAGS = /* @__PURE__ */ new Set([
+  "sequence",
+  "choice",
+  "many",
+  "oneOrMore",
+  "optional",
+  "sepBy",
+  "transform",
+  "node",
+  "not"
+]);
+function isHoistableTag(tag) {
+  return HOISTABLE_TAGS.has(tag);
+}
+var HOIST_MIN_SUBTREE = 3;
 function analyzeLazyUsage(root) {
   return analyzeLazyUsageMulti([root]);
 }
+function subtreeSizes(roots) {
+  const sizes = /* @__PURE__ */ new Map();
+  const sz = (p) => {
+    const cached = sizes.get(p);
+    if (cached !== void 0) return cached;
+    const def = p._def;
+    let s = 1;
+    if (def.tag !== "lazy") {
+      for (const child of childrenOf(def)) s += child._def.tag === "lazy" ? 1 : sz(child);
+    }
+    sizes.set(p, s);
+    return s;
+  };
+  const visitedLazy = /* @__PURE__ */ new Set();
+  const visit = (p) => {
+    const def = p._def;
+    if (def.tag === "lazy") {
+      if (visitedLazy.has(p)) return;
+      visitedLazy.add(p);
+      let resolved;
+      try {
+        resolved = def.thunk();
+      } catch {
+        return;
+      }
+      visit(resolved);
+      return;
+    }
+    sz(p);
+    for (const child of childrenOf(def)) visit(child);
+  };
+  for (const root of roots) visit(root);
+  return sizes;
+}
 function analyzeLazyUsageMulti(roots) {
   const counts = /* @__PURE__ */ new Map();
   const recursive = /* @__PURE__ */ new Set();
@@ -4548,10 +4852,14 @@ function analyzeLazyUsageMulti(roots) {
       active.delete(p);
       return;
     }
+    if (isHoistableTag(def.tag)) {
+      counts.set(p, (counts.get(p) ?? 0) + 1);
+    }
     for (const child of childrenOf(def)) walk2(child);
   }
-  for (const root of roots) walk2(root);
-  return { counts, recursive };
+  const rootsArr = [...roots];
+  for (const root of rootsArr) walk2(root);
+  return { counts, recursive, sizes: subtreeSizes(rootsArr) };
 }
 function ruleDependencies(ruleMap) {
   const nameOf = /* @__PURE__ */ new Map();
@@ -4877,6 +5185,174 @@ function buildInlineExpression(ctx, r, mapFnSources, buildSources) {
   ].join("\n");
 }
 
+// src/combinators/scanTo.ts
+function scanTo(sentinel, { skip: skip2 = [], orEOF = false } = {}) {
+  const meta = {
+    firstSet: any(),
+    canMatchNewline: true,
+    isTrivia: false
+  };
+  return {
+    _tag: "scanTo",
+    _meta: meta,
+    _def: { tag: "scanTo", sentinel, skip: skip2, orEOF },
+    parse(input, pos, ctx) {
+      let cur = pos;
+      const probeCtx = {
+        trackLines: ctx.trackLines,
+        state: ctx.state,
+        ...ctx._errors !== void 0 ? { _errors: ctx._errors } : {}
+      };
+      const emit2 = (end) => {
+        if (end > pos && cstCaptureActive(ctx)) {
+          const leaf = { _tag: "leaf", value: input.slice(pos, end), span: { start: pos, end } };
+          pushCstLeaf(ctx, leaf);
+        }
+      };
+      while (cur < input.length) {
+        const s = sentinel.parse(input, cur, probeCtx);
+        if (s.ok) {
+          emit2(cur);
+          return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
+        }
+        let advanced = false;
+        for (const skipper of skip2) {
+          const r = skipper.parse(input, cur, probeCtx);
+          if (r.ok && r.span.end > cur) {
+            cur = r.span.end;
+            advanced = true;
+            break;
+          }
+        }
+        if (!advanced) cur++;
+      }
+      if (orEOF) {
+        emit2(cur);
+        return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
+      }
+      const sentDef = sentinel._def;
+      const expected = sentDef.tag === "literal" ? [JSON.stringify(sentDef.value)] : ["sentinel"];
+      return { ok: false, expected, span: { start: pos, end: cur } };
+    }
+  };
+}
+function balanced(open, close, options = {}) {
+  const self = ref();
+  const skips = options.skip ?? [];
+  const stop = /* @__PURE__ */ new Set([open, close]);
+  let bounded = true;
+  for (const sk of skips) {
+    const cs = firstSetClassChars(sk);
+    if (cs === null) {
+      bounded = false;
+      break;
+    }
+    for (const ch of cs) stop.add(ch);
+  }
+  const cls = [...stop].map(escapeClassChar).join("");
+  const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
+  const inner = many(choice(self, ...skips, content));
+  const combi = transform(
+    sequence(literal(open), inner, expect(literal(close))),
+    // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
+    // ParseError (recovered close). `c` is the close string or a ParseError.
+    ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
+  );
+  if (combi._def.tag === "transform") {
+    combi._def.fnSrc = '([o, parts, c]) => o + parts.map(p => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")';
+  }
+  self.define(combi);
+  return combi;
+}
+function firstSetClassChars(c) {
+  const fs = c._meta.firstSet;
+  if (fs.kind === "empty") return [];
+  if (fs.kind !== "ranges") return null;
+  const out = [];
+  for (const { lo, hi } of fs.ranges) {
+    if (hi - lo > 8) return null;
+    for (let cp = lo; cp <= hi; cp++) out.push(String.fromCodePoint(cp));
+  }
+  return out;
+}
+function escapeClassChar(ch) {
+  return ch.replace(/[\\\]^-]/g, "\\$&");
+}
+
+// src/compiler/ir-serialize.ts
+function evalRuleMapIR(ir) {
+  const _tf = (child, src) => {
+    let fn2;
+    try {
+      fn2 = (0, eval)(`(${src})`);
+    } catch {
+      fn2 = () => {
+        throw new Error("IR transform fn not materialized");
+      };
+    }
+    const t = transform(child, fn2);
+    t._def.fnSrc = src;
+    return t;
+  };
+  const _nd = (type, child, src, opts) => {
+    const n = node(type, child, void 0, opts);
+    n._def.buildSrc = src;
+    return n;
+  };
+  const fn = new Function(
+    "rules",
+    "ref",
+    "regex",
+    "literal",
+    "keywords",
+    "sequence",
+    "choice",
+    "many",
+    "oneOrMore",
+    "optional",
+    "sepBy",
+    "not",
+    "node",
+    "parser",
+    "scanTo",
+    "token",
+    "transform",
+    "skip",
+    "trivia",
+    "label",
+    "expect",
+    "_tf",
+    "_nd",
+    `return (${ir})`
+  );
+  const map = fn(
+    rules,
+    ref,
+    regex,
+    literal,
+    keywords,
+    sequence,
+    choice,
+    many,
+    oneOrMore,
+    optional,
+    sepBy,
+    not,
+    node,
+    parser,
+    scanTo,
+    token,
+    transform,
+    skip,
+    trivia,
+    label,
+    expect,
+    _tf,
+    _nd
+  );
+  return Object.entries(map);
+}
+
 // src/compiler/linker.ts
 var _nsCounter = 0;
 function linkable(rulesMap, ns) {
@@ -4961,9 +5437,18 @@ function fuseRules(pieces) {
   return new Function("_env", body)(env);
 }
 var COMPOSED_PIECES = /* @__PURE__ */ Symbol.for("parseman.composedPieces");
+function isIRPiece(p) {
+  return !!p && typeof p === "object" && typeof p.ir === "string" && typeof p.ns === "string" && !("ruleFns" in p);
+}
+function materializePiece(p) {
+  if (!isIRPiece(p)) return p;
+  const pieces = compileLinkable(evalRuleMapIR(p.ir), p.ns);
+  if (!pieces) throw new Error(`compose: carried IR for ns "${p.ns}" could not be re-lowered`);
+  return pieces;
+}
 function itemPieces(item) {
   const carried = item[COMPOSED_PIECES];
-  if (Array.isArray(carried)) return carried;
+  if (Array.isArray(carried)) return carried.map(materializePiece);
   if (item.ruleFns instanceof Map) return [item];
   return [linkable(item)];
 }
@@ -5017,7 +5502,7 @@ function recover(combinator, sentinel) {
     canMatchNewline: true,
     isTrivia: false
   };
-  const expected = staticExpected(combinator);
+  const expected = deriveExpected(combinator);
   return {
     _tag: "recover",
     _meta: meta,
@@ -5059,100 +5544,6 @@ function deeperFail(a, b) {
   return a.span.start >= b.span.start ? a : b;
 }
 
-// src/combinators/scanTo.ts
-function scanTo(sentinel, { skip: skip2 = [], orEOF = false } = {}) {
-  const meta = {
-    firstSet: any(),
-    canMatchNewline: true,
-    isTrivia: false
-  };
-  return {
-    _tag: "scanTo",
-    _meta: meta,
-    _def: { tag: "scanTo", sentinel, skip: skip2, orEOF },
-    parse(input, pos, ctx) {
-      let cur = pos;
-      const probeCtx = {
-        trackLines: ctx.trackLines,
-        state: ctx.state,
-        ...ctx._errors !== void 0 ? { _errors: ctx._errors } : {}
-      };
-      const emit2 = (end) => {
-        if (end > pos && cstCaptureActive(ctx)) {
-          const leaf = { _tag: "leaf", value: input.slice(pos, end), span: { start: pos, end } };
-          pushCstLeaf(ctx, leaf);
-        }
-      };
-      while (cur < input.length) {
-        const s = sentinel.parse(input, cur, probeCtx);
-        if (s.ok) {
-          emit2(cur);
-          return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
-        }
-        let advanced = false;
-        for (const skipper of skip2) {
-          const r = skipper.parse(input, cur, probeCtx);
-          if (r.ok && r.span.end > cur) {
-            cur = r.span.end;
-            advanced = true;
-            break;
-          }
-        }
-        if (!advanced) cur++;
-      }
-      if (orEOF) {
-        emit2(cur);
-        return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
-      }
-      const sentDef = sentinel._def;
-      const expected = sentDef.tag === "literal" ? [JSON.stringify(sentDef.value)] : ["sentinel"];
-      return { ok: false, expected, span: { start: pos, end: cur } };
-    }
-  };
-}
-function balanced(open, close, options = {}) {
-  const self = ref();
-  const skips = options.skip ?? [];
-  const stop = /* @__PURE__ */ new Set([open, close]);
-  let bounded = true;
-  for (const sk of skips) {
-    const cs = firstSetClassChars(sk);
-    if (cs === null) {
-      bounded = false;
-      break;
-    }
-    for (const ch of cs) stop.add(ch);
-  }
-  const cls = [...stop].map(escapeClassChar).join("");
-  const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
-  const inner = many(choice(self, ...skips, content));
-  const combi = transform(
-    sequence(literal(open), inner, expect(literal(close), close)),
-    // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
-    // ParseError (recovered close). `c` is the close string or a ParseError.
-    ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
-  );
-  if (combi._def.tag === "transform") {
-    combi._def.fnSrc = '([o, parts, c]) => o + parts.map(p => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")';
-  }
-  self.define(combi);
-  return combi;
-}
-function firstSetClassChars(c) {
-  const fs = c._meta.firstSet;
-  if (fs.kind === "empty") return [];
-  if (fs.kind !== "ranges") return null;
-  const out = [];
-  for (const { lo, hi } of fs.ranges) {
-    if (hi - lo > 8) return null;
-    for (let cp = lo; cp <= hi; cp++) out.push(String.fromCodePoint(cp));
-  }
-  return out;
-}
-function escapeClassChar(ch) {
-  return ch.replace(/[\\\]^-]/g, "\\$&");
-}
-
 // src/combinators/recover-list.ts
 function tolerantElement(item, sentinel, until) {
   const el = transform(