parseman 0.16.0 → 0.17.0

package/dist/index.js

@@ -84,6 +84,54 @@ function sequenceFirstSet(parsers) {
   }
   return fs;
 }
+function firstSetOf(p, seen = /* @__PURE__ */ new Set()) {
+  if (seen.has(p)) return any();
+  seen.add(p);
+  const fs = (c) => firstSetOf(c, seen);
+  const d = p._def;
+  switch (d.tag) {
+    case "literal":
+    case "regex":
+    case "keywords":
+      return p._meta.firstSet;
+    // terminals: no refs, cached set is exact
+    case "lazy":
+      try {
+        return fs(d.thunk());
+      } catch {
+        return any();
+      }
+    case "choice": {
+      let out = empty();
+      for (const arm of d.parsers) out = union(out, fs(arm));
+      return out;
+    }
+    case "sequence": {
+      let out = empty();
+      for (const term of d.parsers) {
+        out = union(out, fs(term));
+        if (!matchesEmpty(term)) return out;
+      }
+      return out;
+    }
+    case "oneOrMore":
+    case "many":
+    case "optional":
+    case "transform":
+    case "label":
+    case "trivia":
+    case "node":
+    case "grammar":
+    case "expect":
+      return fs(d.parser);
+    case "sepBy":
+      return fs(d.parser);
+    case "skip":
+      return fs(d.main);
+    default:
+      return p._meta.firstSet;
+  }
+}
 function mergeRanges(ranges) {
   if (ranges.length === 0) return [];
   const sorted = [...ranges].sort((a, b) => a.lo - b.lo);
@@ -3024,16 +3072,12 @@ function namedFnPrelude() {
 function pushNamedFnDecl(ctx, fnName, bodyStmts, valueVar, endVar) {
   ctx.namedFnDecls.push([
     `function ${fnName}(input, _pos, _ctx) {`,
-    `  let _pfok = false, _pfv, _pfe = _pos`,
     `  _pfail: {`,
     ...reindentStmts(bodyStmts, 2),
-    `    _pfv = ${valueVar}`,
-    `    _pfe = ${endVar}`,
-    `    _pfok = true`,
+    `    ${NAMED_FN_END} = ${endVar}`,
+    `    return ${valueVar}`,
     `  }`,
-    `  if (!_pfok) return ${NAMED_FN_FAIL}`,
-    `  ${NAMED_FN_END} = _pfe`,
-    `  return _pfv`,
+    `  return ${NAMED_FN_FAIL}`,
     `}`
   ].join("\n"));
 }
@@ -3734,11 +3778,16 @@ function emitFirstMatch(def, ctx, pos) {
     const atStart = failsAtStart(p);
     const staticFx = armStaticExpected(ctx, p);
     const deferRuleName = ctx.deferFirstSetRefs ? p._ruleName : void 0;
-    const fsGuard = deferRuleName !== void 0 ? `/*@FS:${deferRuleName}:${codeV}@*/true` : needsFirstSetGuard(p) ? firstSetCond(codeV, p._meta.firstSet) : null;
+    const fsGuard = deferRuleName !== void 0 ? `/*@FS:${deferRuleName}:${codeV}@*/true` : needsFirstSetGuard(p) ? firstSetCond(codeV, p._meta.firstSet) : (() => {
+      if (p._meta.firstSet.kind !== "any" || matchesEmpty(p)) return null;
+      const deep = firstSetOf(p);
+      return deep.kind === "ranges" ? firstSetCond(codeV, deep) : null;
+    })();
     let gateCond = null;
     if (gate) {
       const gateIdx = ctx.mapFns.length;
       ctx.mapFns.push(gate);
+      ctx.mapFnSrcs.push(null);
       gateCond = `${mfRef(ctx)}[${gateIdx}](_ctx.state)`;
     }
     const skipCond = gateCond ? `!${resOkV} && ${gateCond}` : `!${resOkV}`;
@@ -3797,13 +3846,21 @@ function emitNonDisjoint(def, strategy, allExpected, ctx, pos) {
     return emitLiteralsLongestFirst(def, strategy.sortedIndices, allExpected, ctx, pos);
   return emitFirstMatch(def, ctx, pos);
 }
+function pushMapFn(ctx, fn, src) {
+  if (src !== null && ctx.mapFns.length === ctx.mapFnSrcs.length) {
+    const hit = ctx.mapFnSrcs.indexOf(src);
+    if (hit !== -1) return hit;
+  }
+  const idx = ctx.mapFns.length;
+  ctx.mapFns.push(fn);
+  ctx.mapFnSrcs.push(src);
+  return idx;
+}
 function emitTransformChain(p, baseValue, endV, startPos, ctx) {
   const def = p._def;
   if (def.tag === "transform") {
     const innerR = emitTransformChain(def.parser, baseValue, endV, startPos, ctx);
-    const fnIdx = ctx.mapFns.length;
-    ctx.mapFns.push(def.fn);
-    ctx.mapFnSrcs.push(def.fnSrc ?? null);
+    const fnIdx = pushMapFn(ctx, def.fn, def.fnSrc ?? null);
     const vv = v(ctx);
     return {
       stmts: [...innerR.stmts, `${ind(ctx)}const ${vv} = ${mfRef(ctx)}[${fnIdx}](${innerR.valueVar}, { start: ${startPos}, end: ${endV} })`],
@@ -3986,7 +4043,7 @@ function emitScanTo(def, ctx, pos) {
     `${ind(ctx)}while (${curV} < input.length) {`
   ];
   ctx.indent++;
-  const probeCtx = { ...ctx, capturing: false };
+  const probeCtx = { ...ctx, capturing: false, noHoist: true };
   const { stmts: sentStmts, okVar: sentOk } = emitFallible(def.sentinel, probeCtx, curV);
   stmts.push(...sentStmts, `${ind(ctx)}if (${sentOk}) { ${foundV} = true; break }`);
   if (def.skip.length > 0) {
@@ -4177,6 +4234,32 @@ function emitExpect(def, ctx, pos) {
   return { stmts, valueVar: valVar, endVar };
 }
 function emit(p, ctx, pos) {
+  const usage = ctx.lazyUsage;
+  if (usage && !ctx.noHoist && !ctx.capAsTrivia && !ctx.ruleNames?.has(p) && isHoistableTag(p._def.tag) && (usage.counts.get(p) ?? 0) > 1 && // Only hoist subtrees big enough that de-duplicating them beats the call
+  // overhead. A tiny shared wrapper (e.g. `optional(literal(';'))`) inlines
+  // faster than it calls and barely moves the byte count; the size explosion
+  // is all in LARGE choices/sequences referenced from many positions.
+  (usage.sizes.get(p) ?? 0) >= HOIST_MIN_SUBTREE) {
+    const existing = ctx.namedParsers.get(p);
+    if (existing !== void 0) return emitNamedFnCall(ctx, existing, pos);
+    const fnName = `${nsp(ctx)}_pf${ctx.namedParsers.size}`;
+    ctx.namedParsers.set(p, fnName);
+    const savedIndent = ctx.indent;
+    const savedFailLabel = ctx.failLabel;
+    const savedRecord = ctx.recordFail;
+    ctx.indent = 1;
+    ctx.failLabel = "_pfail";
+    ctx.recordFail = true;
+    const r = emitDispatch(p, ctx, "_pos");
+    ctx.indent = savedIndent;
+    ctx.failLabel = savedFailLabel;
+    ctx.recordFail = savedRecord;
+    pushNamedFnDecl(ctx, fnName, r.stmts, r.valueVar, r.endVar);
+    return emitNamedFnCall(ctx, fnName, pos);
+  }
+  return emitDispatch(p, ctx, pos);
+}
+function emitDispatch(p, ctx, pos) {
   const def = p._def;
   switch (def.tag) {
     case "literal":
@@ -4222,9 +4305,7 @@ function emit(p, ctx, pos) {
           };
         }
       }
-      const fnIdx = ctx.mapFns.length;
-      ctx.mapFns.push(def.fn);
-      ctx.mapFnSrcs.push(def.fnSrc ?? null);
+      const fnIdx = pushMapFn(ctx, def.fn, def.fnSrc ?? null);
       const mv = v(ctx, "_mapped");
       return {
         stmts: [
@@ -4289,6 +4370,7 @@ function emit(p, ctx, pos) {
     case "guard": {
       const fnIdx = ctx.mapFns.length;
       ctx.mapFns.push(def.predicate);
+      ctx.mapFnSrcs.push(null);
       const vv = v(ctx);
       return {
         stmts: [
@@ -4303,6 +4385,7 @@ function emit(p, ctx, pos) {
       const evIdx = ctx.mapFns.length;
       const extra = def.extra;
       ctx.mapFns.push((() => extra));
+      ctx.mapFnSrcs.push(null);
       const innerParser = def.parser;
       if (!ctx.namedParsers.has(innerParser)) {
         const fnName = `_wcf${ctx.namedParsers.size}`;
@@ -4422,9 +4505,58 @@ function childrenOf(def) {
       return [];
   }
 }
+var HOISTABLE_TAGS = /* @__PURE__ */ new Set([
+  "sequence",
+  "choice",
+  "many",
+  "oneOrMore",
+  "optional",
+  "sepBy",
+  "transform",
+  "node",
+  "not"
+]);
+function isHoistableTag(tag) {
+  return HOISTABLE_TAGS.has(tag);
+}
+var HOIST_MIN_SUBTREE = 3;
 function analyzeLazyUsage(root) {
   return analyzeLazyUsageMulti([root]);
 }
+function subtreeSizes(roots) {
+  const sizes = /* @__PURE__ */ new Map();
+  const sz = (p) => {
+    const cached = sizes.get(p);
+    if (cached !== void 0) return cached;
+    const def = p._def;
+    let s = 1;
+    if (def.tag !== "lazy") {
+      for (const child of childrenOf(def)) s += child._def.tag === "lazy" ? 1 : sz(child);
+    }
+    sizes.set(p, s);
+    return s;
+  };
+  const visitedLazy = /* @__PURE__ */ new Set();
+  const visit = (p) => {
+    const def = p._def;
+    if (def.tag === "lazy") {
+      if (visitedLazy.has(p)) return;
+      visitedLazy.add(p);
+      let resolved;
+      try {
+        resolved = def.thunk();
+      } catch {
+        return;
+      }
+      visit(resolved);
+      return;
+    }
+    sz(p);
+    for (const child of childrenOf(def)) visit(child);
+  };
+  for (const root of roots) visit(root);
+  return sizes;
+}
 function analyzeLazyUsageMulti(roots) {
   const counts = /* @__PURE__ */ new Map();
   const recursive = /* @__PURE__ */ new Set();
@@ -4451,10 +4583,14 @@ function analyzeLazyUsageMulti(roots) {
       active.delete(p);
       return;
     }
+    if (isHoistableTag(def.tag)) {
+      counts.set(p, (counts.get(p) ?? 0) + 1);
+    }
     for (const child of childrenOf(def)) walk2(child);
   }
-  for (const root of roots) walk2(root);
-  return { counts, recursive };
+  const rootsArr = [...roots];
+  for (const root of rootsArr) walk2(root);
+  return { counts, recursive, sizes: subtreeSizes(rootsArr) };
 }
 function ruleDependencies(ruleMap) {
   const nameOf = /* @__PURE__ */ new Map();
@@ -4780,6 +4916,172 @@ function buildInlineExpression(ctx, r, mapFnSources, buildSources) {
   ].join("\n");
 }
 
+// src/combinators/scanTo.ts
+function scanTo(sentinel, { skip: skip2 = [], orEOF = false } = {}) {
+  const meta = {
+    firstSet: any(),
+    canMatchNewline: true,
+    isTrivia: false
+  };
+  return {
+    _tag: "scanTo",
+    _meta: meta,
+    _def: { tag: "scanTo", sentinel, skip: skip2, orEOF },
+    parse(input, pos, ctx) {
+      let cur = pos;
+      const probeCtx = {
+        trackLines: ctx.trackLines,
+        state: ctx.state,
+        ...ctx._errors !== void 0 ? { _errors: ctx._errors } : {}
+      };
+      const emit2 = (end) => {
+        if (end > pos && cstCaptureActive(ctx)) {
+          const leaf = { _tag: "leaf", value: input.slice(pos, end), span: { start: pos, end } };
+          pushCstLeaf(ctx, leaf);
+        }
+      };
+      while (cur < input.length) {
+        const s = sentinel.parse(input, cur, probeCtx);
+        if (s.ok) {
+          emit2(cur);
+          return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
+        }
+        let advanced = false;
+        for (const skipper of skip2) {
+          const r = skipper.parse(input, cur, probeCtx);
+          if (r.ok && r.span.end > cur) {
+            cur = r.span.end;
+            advanced = true;
+            break;
+          }
+        }
+        if (!advanced) cur++;
+      }
+      if (orEOF) {
+        emit2(cur);
+        return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
+      }
+      const sentDef = sentinel._def;
+      const expected = sentDef.tag === "literal" ? [JSON.stringify(sentDef.value)] : ["sentinel"];
+      return { ok: false, expected, span: { start: pos, end: cur } };
+    }
+  };
+}
+function balanced(open, close, options = {}) {
+  const self = ref();
+  const skips = options.skip ?? [];
+  const stop = /* @__PURE__ */ new Set([open, close]);
+  let bounded = true;
+  for (const sk of skips) {
+    const cs = firstSetClassChars(sk);
+    if (cs === null) {
+      bounded = false;
+      break;
+    }
+    for (const ch of cs) stop.add(ch);
+  }
+  const cls = [...stop].map(escapeClassChar).join("");
+  const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
+  const inner = many(choice(self, ...skips, content));
+  const combi = transform(
+    sequence(literal(open), inner, expect(literal(close), close)),
+    // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
+    // ParseError (recovered close). `c` is the close string or a ParseError.
+    ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
+  );
+  if (combi._def.tag === "transform") {
+    combi._def.fnSrc = '([o, parts, c]) => o + parts.map(p => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")';
+  }
+  self.define(combi);
+  return combi;
+}
+function firstSetClassChars(c) {
+  const fs = c._meta.firstSet;
+  if (fs.kind === "empty") return [];
+  if (fs.kind !== "ranges") return null;
+  const out = [];
+  for (const { lo, hi } of fs.ranges) {
+    if (hi - lo > 8) return null;
+    for (let cp = lo; cp <= hi; cp++) out.push(String.fromCodePoint(cp));
+  }
+  return out;
+}
+function escapeClassChar(ch) {
+  return ch.replace(/[\\\]^-]/g, "\\$&");
+}
+
+// src/compiler/ir-serialize.ts
+function evalRuleMapIR(ir) {
+  const _tf = (child, src) => {
+    let fn2;
+    try {
+      fn2 = (0, eval)(`(${src})`);
+    } catch {
+      fn2 = () => {
+        throw new Error("IR transform fn not materialized");
+      };
+    }
+    const t = transform(child, fn2);
+    t._def.fnSrc = src;
+    return t;
+  };
+  const _nd = (type, child, src, opts) => {
+    const n = node(type, child, void 0, opts);
+    n._def.buildSrc = src;
+    return n;
+  };
+  const fn = new Function(
+    "rules",
+    "ref",
+    "regex",
+    "literal",
+    "keywords",
+    "sequence",
+    "choice",
+    "many",
+    "oneOrMore",
+    "optional",
+    "sepBy",
+    "not",
+    "node",
+    "parser",
+    "scanTo",
+    "transform",
+    "skip",
+    "trivia",
+    "label",
+    "expect",
+    "_tf",
+    "_nd",
+    `return (${ir})`
+  );
+  const map = fn(
+    rules,
+    ref,
+    regex,
+    literal,
+    keywords,
+    sequence,
+    choice,
+    many,
+    oneOrMore,
+    optional,
+    sepBy,
+    not,
+    node,
+    parser,
+    scanTo,
+    transform,
+    skip,
+    trivia,
+    label,
+    expect,
+    _tf,
+    _nd
+  );
+  return Object.entries(map);
+}
+
 // src/compiler/linker.ts
 var _nsCounter = 0;
 function linkable(rulesMap, ns) {
@@ -4864,9 +5166,18 @@ function fuseRules(pieces) {
   return new Function("_env", body)(env);
 }
 var COMPOSED_PIECES = /* @__PURE__ */ Symbol.for("parseman.composedPieces");
+function isIRPiece(p) {
+  return !!p && typeof p === "object" && typeof p.ir === "string" && typeof p.ns === "string" && !("ruleFns" in p);
+}
+function materializePiece(p) {
+  if (!isIRPiece(p)) return p;
+  const pieces = compileLinkable(evalRuleMapIR(p.ir), p.ns);
+  if (!pieces) throw new Error(`compose: carried IR for ns "${p.ns}" could not be re-lowered`);
+  return pieces;
+}
 function itemPieces(item) {
   const carried = item[COMPOSED_PIECES];
-  if (Array.isArray(carried)) return carried;
+  if (Array.isArray(carried)) return carried.map(materializePiece);
   if (item.ruleFns instanceof Map) return [item];
   return [linkable(item)];
 }
@@ -4962,100 +5273,6 @@ function deeperFail(a, b) {
   return a.span.start >= b.span.start ? a : b;
 }
 
-// src/combinators/scanTo.ts
-function scanTo(sentinel, { skip: skip2 = [], orEOF = false } = {}) {
-  const meta = {
-    firstSet: any(),
-    canMatchNewline: true,
-    isTrivia: false
-  };
-  return {
-    _tag: "scanTo",
-    _meta: meta,
-    _def: { tag: "scanTo", sentinel, skip: skip2, orEOF },
-    parse(input, pos, ctx) {
-      let cur = pos;
-      const probeCtx = {
-        trackLines: ctx.trackLines,
-        state: ctx.state,
-        ...ctx._errors !== void 0 ? { _errors: ctx._errors } : {}
-      };
-      const emit2 = (end) => {
-        if (end > pos && cstCaptureActive(ctx)) {
-          const leaf = { _tag: "leaf", value: input.slice(pos, end), span: { start: pos, end } };
-          pushCstLeaf(ctx, leaf);
-        }
-      };
-      while (cur < input.length) {
-        const s = sentinel.parse(input, cur, probeCtx);
-        if (s.ok) {
-          emit2(cur);
-          return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
-        }
-        let advanced = false;
-        for (const skipper of skip2) {
-          const r = skipper.parse(input, cur, probeCtx);
-          if (r.ok && r.span.end > cur) {
-            cur = r.span.end;
-            advanced = true;
-            break;
-          }
-        }
-        if (!advanced) cur++;
-      }
-      if (orEOF) {
-        emit2(cur);
-        return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
-      }
-      const sentDef = sentinel._def;
-      const expected = sentDef.tag === "literal" ? [JSON.stringify(sentDef.value)] : ["sentinel"];
-      return { ok: false, expected, span: { start: pos, end: cur } };
-    }
-  };
-}
-function balanced(open, close, options = {}) {
-  const self = ref();
-  const skips = options.skip ?? [];
-  const stop = /* @__PURE__ */ new Set([open, close]);
-  let bounded = true;
-  for (const sk of skips) {
-    const cs = firstSetClassChars(sk);
-    if (cs === null) {
-      bounded = false;
-      break;
-    }
-    for (const ch of cs) stop.add(ch);
-  }
-  const cls = [...stop].map(escapeClassChar).join("");
-  const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
-  const inner = many(choice(self, ...skips, content));
-  const combi = transform(
-    sequence(literal(open), inner, expect(literal(close), close)),
-    // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
-    // ParseError (recovered close). `c` is the close string or a ParseError.
-    ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
-  );
-  if (combi._def.tag === "transform") {
-    combi._def.fnSrc = '([o, parts, c]) => o + parts.map(p => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")';
-  }
-  self.define(combi);
-  return combi;
-}
-function firstSetClassChars(c) {
-  const fs = c._meta.firstSet;
-  if (fs.kind === "empty") return [];
-  if (fs.kind !== "ranges") return null;
-  const out = [];
-  for (const { lo, hi } of fs.ranges) {
-    if (hi - lo > 8) return null;
-    for (let cp = lo; cp <= hi; cp++) out.push(String.fromCodePoint(cp));
-  }
-  return out;
-}
-function escapeClassChar(ch) {
-  return ch.replace(/[\\\]^-]/g, "\\$&");
-}
-
 // src/combinators/recover-list.ts
 function tolerantElement(item, sentinel, until) {
   const el = transform(