parseman 0.22.0 → 0.24.0

package/dist/plugin/index.cjs

@@ -422,24 +422,35 @@ function literal(value, opts = {}) {
       { lo: firstUpper, hi: firstUpper }
     ] } : firstSet2;
   }
+  const expected = [JSON.stringify(value)];
+  const parse2 = !caseInsensitive && value.length === 1 ? (() => {
+    const code = value.charCodeAt(0);
+    return function parse3(input, pos, ctx) {
+      if (input.charCodeAt(pos) === code) {
+        const span = { start: pos, end: pos + 1 };
+        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value, span });
+        return { ok: true, value, span };
+      }
+      return failAt(ctx, expected, pos);
+    };
+  })() : function parse3(input, pos, ctx) {
+    const end = pos + value.length;
+    if (end > input.length) {
+      return failAt(ctx, expected, pos);
+    }
+    const matchedValue = caseInsensitive ? input.slice(pos, end) : value;
+    if (caseInsensitive ? asciiFoldEq(matchedValue, value) : input.startsWith(value, pos)) {
+      const span = { start: pos, end };
+      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value: matchedValue, span });
+      return { ok: true, value: matchedValue, span };
+    }
+    return failAt(ctx, expected, pos);
+  };
   return {
     _tag: "literal",
     _meta: meta,
     _def: { tag: "literal", value, caseInsensitive },
-    parse(input, pos, ctx) {
-      const end = pos + value.length;
-      if (end > input.length) {
-        return failAt(ctx, [JSON.stringify(value)], pos);
-      }
-      const matchedValue = caseInsensitive ? input.slice(pos, end) : value;
-      if (caseInsensitive ? asciiFoldEq(matchedValue, value) : input.startsWith(value, pos)) {
-        const span = { start: pos, end };
-        const leaf = { _tag: "leaf", value: matchedValue, span };
-        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, leaf);
-        return { ok: true, value: matchedValue, span };
-      }
-      return failAt(ctx, [JSON.stringify(value)], pos);
-    }
+    parse: parse2
   };
 }
 
@@ -574,8 +585,7 @@ function regex(pattern, flags = "") {
         if (scanEnd === null) return failAt(ctx, [`/${source}/`], pos);
         const value = input.slice(pos, scanEnd);
         const span2 = { start: pos, end: scanEnd };
-        const leaf2 = { _tag: "leaf", value, span: span2 };
-        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, leaf2);
+        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value, span: span2 });
         return { ok: true, value, span: span2 };
       }
       anchored.lastIndex = pos;
@@ -584,8 +594,7 @@ function regex(pattern, flags = "") {
         return failAt(ctx, [`/${source}/`], pos);
       }
       const span = { start: pos, end: pos + m[0].length };
-      const leaf = { _tag: "leaf", value: m[0], span };
-      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, leaf);
+      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value: m[0], span });
       return { ok: true, value: m[0], span };
     }
   };
@@ -890,8 +899,7 @@ function keywords(words, opts = {}) {
       }
       const value = m[0];
       const span = { start: pos, end: pos + value.length };
-      const leaf = { _tag: "leaf", value, span };
-      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, leaf);
+      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value, span });
       return { ok: true, value, span };
     }
   };
@@ -1381,33 +1389,136 @@ function loopScanner(arms) {
 }
 function regexTriviaScanner(parser2) {
   if (parser2._def.tag !== "regex" || parser2._def.flags) return null;
-  const ws = whitespaceSource(parser2._def.source);
-  if (ws) return ws;
-  return blockCommentSource(parser2._def.source) ? scanBlockComment : null;
+  const source = parser2._def.source;
+  return classRunSource(source) ?? altStarSource(source) ?? (blockCommentSource(source) ? scanBlockComment : null);
 }
-function whitespaceSource(source) {
-  const m = /^\[([ \\tnrf]+)\][*+]$/.exec(source);
-  if (!m) return null;
-  const chars = /* @__PURE__ */ new Set();
-  for (let i = 0; i < m[1].length; i++) {
-    const ch = m[1][i];
-    if (ch !== "\\") {
-      chars.add(ch.charCodeAt(0));
-      continue;
+function inRanges2(cp, ranges) {
+  for (let i = 0; i < ranges.length; i++) {
+    const r = ranges[i];
+    if (cp >= r[0] && cp <= r[1]) return true;
+  }
+  return false;
+}
+function classScanner(classBody) {
+  const ranges = parseClassRanges(classBody);
+  if (!ranges) return null;
+  return (input, cur) => {
+    let pos = cur;
+    while (pos < input.length && inRanges2(input.charCodeAt(pos), ranges)) pos++;
+    return pos;
+  };
+}
+function lineCommentScanner(commentCode) {
+  return (input, cur) => {
+    if (input.charCodeAt(cur) !== commentCode) return cur;
+    let pos = cur + 1;
+    while (pos < input.length) {
+      const cc = input.charCodeAt(pos);
+      if (cc === 10 || cc === 13) break;
+      pos++;
     }
-    const esc = m[1][++i];
-    if (esc === "t") chars.add(9);
-    else if (esc === "n") chars.add(10);
-    else if (esc === "r") chars.add(13);
-    else if (esc === "f") chars.add(12);
-    else return null;
+    return pos;
+  };
+}
+function classRunSource(source) {
+  const m = /^\[([^\]^](?:[^\]]|\\.)*)\][*+]$/.exec(source);
+  return m ? classScanner(m[1]) : null;
+}
+function classifyTriviaArm(arm) {
+  const cls = /^\[([^\]^](?:[^\]]|\\.)*)\][*+]?$/.exec(arm);
+  if (cls) {
+    const ranges = parseClassRanges(cls[1]);
+    return ranges ? { kind: "class", ranges } : null;
+  }
+  const lc = /^(\\?.)\[\^\\n\\r\]\*$/.exec(arm);
+  if (lc) {
+    const marker = lc[1];
+    return { kind: "comment", code: (marker.length === 2 ? marker[1] : marker[0]).charCodeAt(0) };
   }
+  return null;
+}
+function armScanner(arm) {
+  if (arm.kind === "comment") return lineCommentScanner(arm.code);
+  const ranges = arm.ranges;
   return (input, cur) => {
     let pos = cur;
-    while (pos < input.length && chars.has(input.charCodeAt(pos))) pos++;
+    while (pos < input.length && inRanges2(input.charCodeAt(pos), ranges)) pos++;
     return pos;
   };
 }
+function fusedTriviaScanner(ranges, commentCodes) {
+  const c0 = commentCodes[0];
+  const single = commentCodes.length === 1;
+  return (input, cur) => {
+    let pos = cur;
+    const len = input.length;
+    for (; ; ) {
+      const c = input.charCodeAt(pos);
+      if (pos < len && inRanges2(c, ranges)) {
+        pos++;
+        continue;
+      }
+      if (single ? c === c0 : commentCodes.includes(c)) {
+        pos++;
+        while (pos < len) {
+          const cc = input.charCodeAt(pos);
+          if (cc === 10 || cc === 13) break;
+          pos++;
+        }
+        continue;
+      }
+      break;
+    }
+    return pos;
+  };
+}
+function splitTopLevelAlts(body) {
+  const arms = [];
+  let inClass = false;
+  let start = 0;
+  for (let i = 0; i < body.length; i++) {
+    const c = body[i];
+    if (c === "\\") {
+      i++;
+      continue;
+    }
+    if (inClass) {
+      if (c === "]") inClass = false;
+      continue;
+    }
+    if (c === "[") inClass = true;
+    else if (c === "(") return null;
+    else if (c === "|") {
+      arms.push(body.slice(start, i));
+      start = i + 1;
+    }
+  }
+  arms.push(body.slice(start));
+  return arms;
+}
+function altStarSource(source) {
+  const m = /^\(\?:(.*)\)[*+]$/.exec(source);
+  if (!m) return null;
+  const armSrcs = splitTopLevelAlts(m[1]);
+  if (!armSrcs || armSrcs.length < 2) return null;
+  const arms = [];
+  for (const src of armSrcs) {
+    const arm = classifyTriviaArm(src);
+    if (!arm) return null;
+    arms.push(arm);
+  }
+  const ranges = [];
+  const commentCodes = [];
+  for (const arm of arms) {
+    if (arm.kind === "class") ranges.push(...arm.ranges);
+    else commentCodes.push(arm.code);
+  }
+  if (commentCodes.some((code) => inRanges2(code, ranges))) {
+    return loopScanner(arms.map(armScanner));
+  }
+  if (commentCodes.length === 0) return armScanner({ kind: "class", ranges });
+  return fusedTriviaScanner(ranges, commentCodes);
+}
 function blockCommentSource(source) {
   return source === "\\/\\*(?:[^*]|\\*(?!\\/))*\\*\\/" || source === "\\/\\*[^]*?\\*\\/";
 }
@@ -1724,6 +1835,7 @@ function markUnusedValues(root) {
 }
 
 // src/combinators/parser.ts
+var RULE_ORDER = "__parsemanRuleOrder";
 function tagRule(r, key) {
   ;
   r._ruleName = key;
@@ -1768,6 +1880,11 @@ function rules(a, b) {
   for (const key of Object.keys(definitions)) {
     markUnusedValues(cache[key]);
   }
+  Object.defineProperty(cache, RULE_ORDER, {
+    value: Object.keys(definitions),
+    enumerable: false,
+    configurable: true
+  });
   return cache;
 }
 
@@ -5539,9 +5656,10 @@ function compileRuleMap(ruleMap, opts) {
   ].join("\n");
   return { keys: perEntry.map((e) => e.key), replacement };
 }
-function compileLinkable(ruleMapArg, ns) {
+function compileLinkable(ruleMapArg, ns, opts) {
   if (!ns) throw new Error("compileLinkable: ns must be a non-empty namespace");
   for (const [, rule] of ruleMapArg) markUnusedValues(rule);
+  const grammarTrivia = opts?.trivia ?? ruleMapArg.map(([, r]) => r._meta.grammarTrivia).find(Boolean);
   const ruleMap = ruleMapArg.filter(([, val]) => {
     const d = val._def;
     if (d.tag !== "lazy") return true;
@@ -5572,7 +5690,8 @@ function compileLinkable(ruleMapArg, ns) {
     capturing: ruleMap.some(([, rule]) => hasNodeDef(rule)),
     lazyUsage: analyzeLazyUsageMulti(ruleMap.map(([, rule]) => rule)),
     ns,
-    deferFirstSetRefs: true
+    deferFirstSetRefs: true,
+    ...grammarTrivia ? { activeTrivia: grammarTrivia, triviaKindLabels: grammarTrivia._meta.triviaKindLabels } : {}
   };
   const seen = /* @__PURE__ */ new Map();
   const ruleNames = /* @__PURE__ */ new Map();
@@ -5635,7 +5754,10 @@ function compileLinkable(ruleMapArg, ns) {
       ctx.indent = 1;
       ctx.failLabel = "_pfail";
       ctx.recordFail = true;
+      const savedTrivia = ctx.activeTrivia;
+      if (rule._meta.isTrivia) ctx.activeTrivia = void 0;
       const body = emit(resolved, ctx, "_pos");
+      ctx.activeTrivia = savedTrivia;
       ctx.indent = savedIndent;
       ctx.failLabel = savedFail;
       ctx.recordFail = savedRec;
@@ -6129,6 +6251,23 @@ ${body}
 };
 
 // src/compiler/linker.ts
+function pickPieces(pieces, names) {
+  const winner = /* @__PURE__ */ new Map();
+  for (const pc of pieces) for (const k of pc.keys) winner.set(k, pc);
+  for (const n of names) {
+    if (!winner.has(n)) throw new Error(`pick: rule "${n}" is not in this grammar (available: ${[...winner.keys()].join(", ")})`);
+  }
+  const keep = /* @__PURE__ */ new Set();
+  const visit = (n) => {
+    const w = winner.get(n);
+    if (keep.has(n) || !w) return;
+    keep.add(n);
+    for (const d of w.deps.get(n) ?? []) visit(d);
+  };
+  for (const n of names) visit(n);
+  const filt = (m, pc) => new Map([...m].filter(([k]) => keep.has(k) && winner.get(k) === pc));
+  return pieces.map((pc) => ({ ...pc, keys: pc.keys.filter((k) => keep.has(k) && winner.get(k) === pc), ruleFns: filt(pc.ruleFns, pc), wrappers: filt(pc.wrappers, pc), deps: filt(pc.deps, pc) })).filter((pc) => pc.keys.length > 0);
+}
 function fusedBody(pieces) {
   const winner = /* @__PURE__ */ new Map();
   for (const p of pieces) for (const k of p.keys) winner.set(k, p);
@@ -6148,7 +6287,7 @@ function fusedBody(pieces) {
     ...needsEmptyTl ? ["const _EMPTY_TL = Object.freeze([])"] : [],
     ...needsHostReads ? [HOST_READS_DECL] : [],
     // Each contributing artifact's namespaced private prelude (regexes, _pf, …).
-    ...[...new Set(contributingPieces.flatMap((p) => p.prelude))],
+    ...new Set(contributingPieces.flatMap((p) => p.prelude)),
     // The winning `_r_<Name>` function for each rule (one per name → no redeclare).
     ...[...winner].map(([k, p]) => p.ruleFns.get(k))
   ];
@@ -6189,9 +6328,9 @@ ${body}
 function isIRPiece(p) {
   return !!p && typeof p === "object" && typeof p.ir === "string" && typeof p.ns === "string" && !("ruleFns" in p);
 }
-function materializePiece(p) {
+function materializePiece(p, trivia2) {
   if (!isIRPiece(p)) return p;
-  const pieces = compileLinkable(evalRuleMapIR(p.ir), p.ns);
+  const pieces = compileLinkable(evalRuleMapIR(p.ir), p.ns, trivia2 ? { trivia: trivia2 } : void 0);
   if (!pieces) throw new Error(`compose: carried IR for ns "${p.ns}" could not be re-lowered`);
   return pieces;
 }
@@ -6897,11 +7036,13 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
       warn(init.start, `${label2}: rule map couldn't be inlined`);
       return null;
     }
-    return { replacement: compiled.replacement, ruleMap };
+    return { replacement: compiled.replacement, ruleMap, ...gTrivia ? { trivia: gTrivia } : {} };
   };
   const isRulesCall = (init) => init.type === "CallExpression" && init.callee.type === "Identifier" && init.callee.name === "rules";
   const isComposeCall = (init) => init.type === "CallExpression" && init.callee.type === "Identifier" && init.callee.name === "compose";
+  const isPickCall = (init) => init.type === "CallExpression" && init.callee.type === "Identifier" && init.callee.name === "pick";
   const localRuleMaps = /* @__PURE__ */ new Map();
+  const localGrammarTrivia = /* @__PURE__ */ new Map();
   const nsFor = (label2) => `_${(0, import_node_crypto.createHash)("sha1").update(`${id}#${label2}`).digest("hex").slice(0, 8)}_`;
   const stripIndent = (s) => s.replace(/\n[ \t]+/g, "\n");
   const serializePieces = (p) => {
@@ -6915,7 +7056,8 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
   const serializeItem = (it) => isSpread(it) ? `...(${it.__spreadLocal}[Symbol.for('parseman.composedPieces')] ?? [])` : isIR(it) ? `{ ns: ${JSON.stringify(it.ns)}, ir: ${JSON.stringify(it.ir)} }` : serializePieces(it);
   const serializeList = (list) => `[${list.map(serializeItem).join(", ")}]`;
   const withCarriedPieces = (grammarExpr, list) => `/* @__PURE__ */ Object.defineProperty(${grammarExpr}, Symbol.for('parseman.composedPieces'), { value: ${serializeList(list)}, enumerable: false })`;
-  const localComposedPieces = /* @__PURE__ */ new Map();
+  const localComposedCarried = /* @__PURE__ */ new Map();
+  const localComposedTrivia = /* @__PURE__ */ new Map();
   const importedPiecesCache = /* @__PURE__ */ new Map();
   const resolveModulePieces = (file, exportName, seen) => {
     const mod = parseModuleCached(file);
@@ -6949,7 +7091,7 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
     }
     try {
       const list = new Function(...stubNames, `return (${literal2})`)(...stubVals);
-      return list.map(materializePiece);
+      return list;
     } catch {
       return null;
     }
@@ -6969,32 +7111,108 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
     importedPiecesCache.set(localName, result2);
     return result2;
   };
-  const localArg = (rm, label2) => {
+  const materializeCarried = (items, composing) => {
+    const out = [];
+    for (const it of items) {
+      if (isSpread(it)) {
+        for (const p of importedPieces(it.__spreadLocal) ?? []) out.push(materializePiece(p, composing));
+      } else if (isIR(it)) {
+        out.push(materializePiece(it, composing));
+      } else {
+        out.push(it);
+      }
+    }
+    return out;
+  };
+  const localCarried = (rm, label2, composing) => {
     const entries = [...rm];
     const ns = nsFor(label2);
-    const p = compileLinkable(entries, ns);
-    if (!p) return null;
     const ir = serializeRuleMap(entries);
-    return ir ? { pieces: [p], ir: { ns, ir } } : { pieces: [p] };
+    if (ir) return { carried: [{ ns, ir }] };
+    const p = compileLinkable(entries, ns, composing ? { trivia: composing } : void 0);
+    return p ? { carried: [p] } : null;
   };
-  const argPieces = (arg, label2) => {
+  const argPieces = (arg, label2, composing) => {
+    if (isPickCall(arg)) {
+      const pargs = arg.arguments;
+      const inner = pargs[0];
+      const namesArg = pargs[1];
+      if (!inner || namesArg?.type !== "ArrayExpression") return null;
+      const names = [];
+      for (const el of namesArg.elements ?? []) {
+        const v2 = el?.value;
+        if ((el?.type === "Literal" || el?.type === "StringLiteral") && typeof v2 === "string") names.push(v2);
+        else return null;
+      }
+      const innerArg = argPieces(inner, `${label2}_pick`);
+      if (!innerArg) return null;
+      try {
+        return { carried: pickPieces(materializeCarried(innerArg.carried, ownTrivia(inner)), names) };
+      } catch (e) {
+        warn(arg.start, `pick(): ${e.message}`);
+        return null;
+      }
+    }
     if (isRulesCall(arg)) {
-      const factory = arg.arguments[0];
+      const rulesArgs = arg.arguments;
+      const a0 = rulesArgs[0];
+      const a1 = rulesArgs[1];
+      const optionsFirst = a0?.type === "ObjectExpression";
+      const factory = optionsFirst ? a1 : a0;
       const rm = factory ? evaluateParserFactory(factory, scope, code, []) : null;
-      return rm ? localArg(rm, label2) : null;
+      return rm ? localCarried(rm, label2, composing) : null;
     }
     if (arg.type === "Identifier") {
       const name = arg.name;
       const rm = localRuleMaps.get(name);
-      if (rm) return localArg(rm, label2);
-      const composed = localComposedPieces.get(name);
-      if (composed) return { pieces: composed };
+      if (rm) return localCarried(rm, label2, composing);
+      const composed = localComposedCarried.get(name);
+      if (composed) return { carried: composed };
       const pieces = importedPieces(name);
       if (!pieces) return null;
-      return importBindings.has(name) ? { pieces, spreadLocal: name } : { pieces };
+      return { carried: importBindings.has(name) ? [{ __spreadLocal: name }] : [...pieces] };
     }
     return null;
   };
+  const rulesCallTrivia = (el) => {
+    const rulesArgs = el.arguments;
+    const a0 = rulesArgs[0];
+    const a1 = rulesArgs[1];
+    const optExpr = a0?.type === "ObjectExpression" ? a0 : a1?.type === "ObjectExpression" ? a1 : void 0;
+    const triviaProp = (optExpr?.properties ?? []).find(
+      (p) => p.key?.name === "trivia"
+    );
+    if (!triviaProp?.value) return void 0;
+    return evaluateExpr(triviaProp.value, scope, code, []) ?? void 0;
+  };
+  const composingTrivia = (elements) => {
+    for (let i = elements.length - 1; i >= 0; i--) {
+      const el = elements[i];
+      if (!el) continue;
+      if (isRulesCall(el)) {
+        const t = rulesCallTrivia(el);
+        if (t) return t;
+        continue;
+      }
+      if (el.type === "Identifier") {
+        const t = localGrammarTrivia.get(el.name);
+        if (t) return t;
+      }
+    }
+    return void 0;
+  };
+  const ownTrivia = (arg) => {
+    if (isRulesCall(arg)) return rulesCallTrivia(arg);
+    if (isPickCall(arg)) {
+      const inner = arg.arguments[0];
+      return inner ? ownTrivia(inner) : void 0;
+    }
+    if (arg.type === "Identifier") {
+      const name = arg.name;
+      return localGrammarTrivia.get(name) ?? localComposedTrivia.get(name);
+    }
+    return void 0;
+  };
   const compileComposeCall = (init) => {
     const args = init.arguments;
     const arr = args[0];
@@ -7003,21 +7221,19 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
       return null;
     }
     const elements = arr.elements;
-    const pieces = [];
+    const composing = composingTrivia(elements);
     const carried = [];
     for (let i = 0; i < elements.length; i++) {
-      const r = argPieces(elements[i], `compose${init.start}_${i}`);
+      const r = argPieces(elements[i], `compose${init.start}_${i}`, composing);
       if (!r) {
         warn(init.start, `compose(): argument ${i} isn't a build-resolvable grammar; falling back to runtime`);
         return null;
       }
-      pieces.push(...r.pieces);
-      if (r.spreadLocal) carried.push({ __spreadLocal: r.spreadLocal });
-      else if (r.ir) carried.push(r.ir);
-      else carried.push(...r.pieces);
+      carried.push(...r.carried);
     }
+    const pieces = materializeCarried(carried, composing);
     try {
-      return { replacement: emitFusedSource(pieces), pieces, carried };
+      return { replacement: emitFusedSource(pieces), carried, ...composing ? { trivia: composing } : {} };
     } catch (e) {
       warn(init.start, `compose(): ${e.message}; falling back to runtime`);
       return null;
@@ -7102,6 +7318,7 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
           const compiledRules = compileRulesFactory(init, varName);
           if (!compiledRules) continue;
           localRuleMaps.set(varName, compiledRules.ruleMap);
+          if (compiledRules.trivia) localGrammarTrivia.set(varName, compiledRules.trivia);
           let replacement = compiledRules.replacement;
           if (exportPrefix) {
             const ns = nsFor(varName);
@@ -7117,7 +7334,8 @@ function transformMacro(code, id, moduleAliases = /* @__PURE__ */ new Set([PARSE
         if (isComposeCall(init)) {
           const fused = compileComposeCall(init);
           if (!fused) continue;
-          localComposedPieces.set(varName, fused.pieces);
+          localComposedCarried.set(varName, fused.carried);
+          if (fused.trivia) localComposedTrivia.set(varName, fused.trivia);
           const replacement = exportPrefix ? withCarriedPieces(fused.replacement, fused.carried) : fused.replacement;
           replacements.push({ start: init.start, end: init.end, replacement });
           continue;