parseman 0.17.0 → 0.18.1

package/dist/plugin/index.js

@@ -65,6 +65,7 @@ function matchesEmpty(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "expect":
     case "withCtx":
     case "node":
@@ -127,6 +128,7 @@ function firstSetOf(p, seen = /* @__PURE__ */ new Set()) {
     case "transform":
     case "label":
     case "trivia":
+    case "token":
     case "node":
     case "grammar":
     case "expect":
@@ -1250,6 +1252,7 @@ function markUnusedValues(root) {
         return;
       case "label":
       case "trivia":
+      case "token":
       case "withCtx":
       case "expect":
         visit(def.parser, consumed);
@@ -1451,7 +1454,11 @@ function label(name, combinator) {
     _tag: combinator._tag,
     _meta: combinator._meta,
     _def: { tag: "label", label: name, parser: combinator },
-    parse: combinator.parse.bind(combinator)
+    parse(input, pos, ctx) {
+      const result = combinator.parse(input, pos, ctx);
+      if (!result.ok) return { ...result, expected: [name] };
+      return result;
+    }
   };
 }
 
@@ -1527,8 +1534,60 @@ function noTrivia(root) {
   return parser({ trivia: null }, root);
 }
 
+// src/combinators/token.ts
+function token(root) {
+  const meta = {
+    firstSet: root._meta.firstSet,
+    canMatchNewline: root._meta.canMatchNewline,
+    isTrivia: false
+  };
+  return {
+    _tag: "token",
+    _meta: meta,
+    _def: { tag: "token", parser: root },
+    parse(input, pos, ctx) {
+      const savedTrivia = ctx.trivia;
+      const savedKinds = ctx.triviaKindLabels;
+      const savedBuf = ctx._cstBuf;
+      const savedChildren = ctx._cstChildren;
+      const savedLeaves = ctx._cstLeaves;
+      const savedRaw = ctx._cstRawChildren;
+      const savedTriviaLog = ctx._cstTriviaLog;
+      const savedOuterTriviaLog = ctx._triviaLog;
+      const wasCapturing = cstCaptureActive(ctx);
+      ctx.trivia = void 0;
+      ctx.triviaKindLabels = void 0;
+      ctx._cstBuf = void 0;
+      ctx._cstChildren = void 0;
+      ctx._cstLeaves = void 0;
+      ctx._cstRawChildren = void 0;
+      ctx._cstTriviaLog = void 0;
+      delete ctx._triviaLog;
+      let result;
+      try {
+        result = root.parse(input, pos, ctx);
+      } finally {
+        ctx.trivia = savedTrivia;
+        ctx.triviaKindLabels = savedKinds;
+        ctx._cstBuf = savedBuf;
+        ctx._cstChildren = savedChildren;
+        ctx._cstLeaves = savedLeaves;
+        ctx._cstRawChildren = savedRaw;
+        ctx._cstTriviaLog = savedTriviaLog;
+        if (savedOuterTriviaLog === void 0) delete ctx._triviaLog;
+        else ctx._triviaLog = savedOuterTriviaLog;
+      }
+      if (!result.ok) return result;
+      const span = result.span;
+      const value = input.slice(pos, span.end);
+      if (wasCapturing) pushCstLeaf(ctx, { _tag: "leaf", value, span });
+      return { ok: true, value, span };
+    }
+  };
+}
+
 // src/combinators/expect.ts
-function staticExpected(c) {
+function deriveExpected(c) {
   const def = c._def;
   switch (def.tag) {
     case "literal":
@@ -1540,22 +1599,23 @@ function staticExpected(c) {
     case "label":
       return [def.label];
     case "choice":
-      return def.parsers.flatMap(staticExpected);
+      return def.parsers.flatMap(deriveExpected);
     case "sequence":
-      return def.parsers.length > 0 ? staticExpected(def.parsers[0]) : [];
+      return def.parsers.length > 0 ? deriveExpected(def.parsers[0]) : [];
     case "node":
     case "grammar":
     case "trivia":
+    case "token":
     case "optional":
     case "many":
     case "oneOrMore":
     case "transform":
     case "not":
-      return staticExpected(def.parser);
+      return deriveExpected(def.parser);
     case "lazy": {
       const name = c._ruleName;
       try {
-        return staticExpected(def.thunk());
+        return deriveExpected(def.thunk());
       } catch {
         return name ? [name] : [];
       }
@@ -1565,7 +1625,7 @@ function staticExpected(c) {
   }
 }
 function expect(combinator, label2) {
-  const expected = label2 !== void 0 ? [label2] : staticExpected(combinator);
+  const expected = label2 !== void 0 ? [label2] : deriveExpected(combinator);
   const meta = {
     firstSet: combinator._meta.firstSet,
     canMatchNewline: combinator._meta.canMatchNewline,
@@ -3225,6 +3285,7 @@ function mayLeavePartialCapture(p, seen = /* @__PURE__ */ new Set()) {
     case "guard":
     case "not":
     case "trivia":
+    case "token":
     case "scanTo":
     case "unknown":
       return false;
@@ -3286,6 +3347,7 @@ function capturesLeaf(p, seen = /* @__PURE__ */ new Set()) {
     case "regex":
     case "keywords":
     case "node":
+    case "token":
       return true;
     case "not":
     case "guard":
@@ -3564,9 +3626,9 @@ function emitSeq(def, ctx, pos) {
   stmts.push(`${ind(ctx)}const ${arrV} = [${valueVars.join(", ")}]`);
   return { stmts, valueVar: arrV, endVar };
 }
-function staticExpectedArr(parsers) {
+function deriveExpectedArr(parsers) {
   return JSON.stringify(parsers.flatMap((p) => {
-    const e = staticExpected(p);
+    const e = deriveExpected(p);
     return e.length > 0 ? e : [p._tag];
   }));
 }
@@ -3587,7 +3649,7 @@ function failsAtStart(p) {
   }
 }
 function armStaticExpected(ctx, p) {
-  return hoistExpected(ctx, staticExpectedArr([p]));
+  return hoistExpected(ctx, deriveExpectedArr([p]));
 }
 function canMatchEmptyAtStart(p) {
   const d = p._def;
@@ -3625,7 +3687,7 @@ function emitDisjointArm(p, ctx, pos, valV, endV) {
   ];
 }
 function emitChoice(def, ctx, pos) {
-  const allExpected = staticExpectedArr(def.parsers);
+  const allExpected = deriveExpectedArr(def.parsers);
   if (def.disjoint) {
     const codeV = v(ctx, "_code");
     const valV = v(ctx, "_chv");
@@ -3686,7 +3748,7 @@ function emitGreedyClassify(def, superIndex, ctx, pos) {
   const wordV = v(ctx, "_gw");
   const endV = v(ctx, "_ge");
   const valV = v(ctx, "_gcv");
-  const regexExpected = JSON.stringify(staticExpected(superParser));
+  const regexExpected = JSON.stringify(deriveExpected(superParser));
   const stmts = [
     `${ind(ctx)}${reVar}.lastIndex = ${pos}`,
     `${ind(ctx)}const ${matchV} = ${reVar}.exec(input)`,
@@ -4078,6 +4140,116 @@ function emitNot(def, ctx, pos) {
     endVar: pos
   };
 }
+function escapeTokenLiteral(value) {
+  return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+}
+function regexMatchesEmpty(source, flags) {
+  try {
+    return new RegExp(`^(?:${source})$`, flags).test("");
+  } catch {
+    return true;
+  }
+}
+function tokenTerminalSource(p) {
+  const def = p._def;
+  switch (def.tag) {
+    case "literal":
+      if (def.value.length === 0) return null;
+      return {
+        source: escapeTokenLiteral(def.value),
+        caseInsensitive: def.caseInsensitive,
+        rawCaseSensitive: def.caseInsensitive ? "" : def.value
+      };
+    case "regex": {
+      if (def.flags.replace(/i/g, "") !== "") return null;
+      if (regexMatchesEmpty(def.source, def.flags)) return null;
+      return {
+        source: `(?:${def.source})`,
+        caseInsensitive: def.flags.includes("i"),
+        rawCaseSensitive: def.flags.includes("i") ? "" : def.source
+      };
+    }
+    default:
+      return null;
+  }
+}
+function tokenSequenceSource(p) {
+  const def = p._def;
+  const parts = def.tag === "sequence" ? def.parsers.map(tokenTerminalSource) : [tokenTerminalSource(p)];
+  if (parts.some((part) => part === null)) return null;
+  const typed = parts;
+  const flags = typed.some((part) => part.caseInsensitive) ? "i" : "";
+  if (flags.includes("i") && typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))) return null;
+  return {
+    source: typed.map((part) => part.source).join(""),
+    flags,
+    hasCaseSensitiveLetters: typed.some((part) => /[A-Za-z]/.test(part.rawCaseSensitive))
+  };
+}
+function combineTokenRegexSources(parts) {
+  const flags = parts.some((part) => part.flags.includes("i")) ? "i" : "";
+  if (flags.includes("i") && parts.some((part) => part.flags === "" && part.hasCaseSensitiveLetters)) return null;
+  return { source: parts.map((part) => part.source).join(""), flags };
+}
+function tokenNullableRegex(def) {
+  const inner = def.parser._def;
+  if (inner.tag === "many") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})*`, flags: seq.flags };
+  }
+  if (inner.tag === "optional") {
+    const seq = tokenSequenceSource(inner.parser);
+    if (!seq) return null;
+    return { tag: "regex", source: `(?:${seq.source})?`, flags: seq.flags };
+  }
+  if (inner.tag === "sepBy") {
+    const item = tokenSequenceSource(inner.parser);
+    const sep = tokenSequenceSource(inner.separator);
+    if (!item || !sep) return null;
+    const combined = combineTokenRegexSources([item, sep]);
+    if (!combined) return null;
+    return { tag: "regex", source: `(?:${item.source}(?:${sep.source}${item.source})*)?`, flags: combined.flags };
+  }
+  return null;
+}
+function emitToken(def, ctx, pos) {
+  const lowered = tokenNullableRegex(def);
+  if (lowered) return emitRegex(lowered, ctx, pos);
+  const savedTrivia = ctx.activeTrivia;
+  const savedKindLabels = ctx.triviaKindLabels;
+  ctx.activeTrivia = void 0;
+  ctx.triviaKindLabels = void 0;
+  const inner = emitFallible(def.parser, ctx, pos);
+  ctx.activeTrivia = savedTrivia;
+  ctx.triviaKindLabels = savedKindLabels;
+  const i = ind(ctx);
+  const sc = v(ctx, "_tokCh");
+  const sl = v(ctx, "_tokLv");
+  const sr = v(ctx, "_tokRaw");
+  const sv = v(ctx, "_tokTr");
+  const sk = v(ctx, "_tokKinds");
+  const stl = v(ctx, "_tokTl");
+  const sol = v(ctx, "_tokLog");
+  const sb = v(ctx, "_tokBuf");
+  const valV = v(ctx, "_tok");
+  return {
+    stmts: [
+      `${i}const ${sc} = _ctx._cstChildren, ${sl} = _ctx._cstLeaves, ${sr} = _ctx._cstRawChildren, ${sv} = _ctx.trivia, ${sk} = _ctx.triviaKindLabels, ${stl} = _ctx._cstTriviaLog, ${sol} = _ctx._triviaLog, ${sb} = _ctx._cstBuf`,
+      `${i}_ctx.trivia = undefined; _ctx.triviaKindLabels = undefined; _ctx._cstChildren = undefined; _ctx._cstLeaves = undefined; _ctx._cstRawChildren = undefined; _ctx._cstTriviaLog = undefined; _ctx._triviaLog = undefined; _ctx._cstBuf = undefined`,
+      `${i}try {`,
+      ...reindentStmts(inner.stmts, ctx.indent + 1).map((stmt) => stmt.replace(/^(\s*)(?:const|let)\s+/, "$1var ")),
+      `${i}} finally {`,
+      `${i}  _ctx.trivia = ${sv}; _ctx.triviaKindLabels = ${sk}; _ctx._cstChildren = ${sc}; _ctx._cstLeaves = ${sl}; _ctx._cstRawChildren = ${sr}; _ctx._cstTriviaLog = ${stl}; _ctx._triviaLog = ${sol}; _ctx._cstBuf = ${sb}`,
+      `${i}}`,
+      ...emitIfFail(ctx, `!${inner.okVar}`, propagateFailBody(ctx)),
+      `${i}const ${valV} = input.slice(${pos}, ${inner.endVar})`,
+      ...emitLeafCapture(ctx, valV, pos, inner.endVar)
+    ],
+    valueVar: valV,
+    endVar: inner.endVar
+  };
+}
 function emitNode(def, ctx, pos) {
   const structural = def.build === void 0;
   const mkType = structural ? null : analyzeMkInlineBuild(def);
@@ -4197,7 +4369,7 @@ function emitRecover(def, ctx, pos) {
     `${ind0}    if (${sentOk}) break`,
     `${ind0}    ${scanV}++`,
     `${ind0}  }`,
-    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(staticExpected(def.parser))} }`,
+    `${ind0}  const ${errV} = { _tag: 'parseError', span: { start: ${pos}, end: ${scanV} }, expected: ${JSON.stringify(deriveExpected(def.parser))} }`,
     `${ind0}  if (_ctx._errors) _ctx._errors.push(${errV})`,
     `${ind0}  ${valVar} = ${errV}`,
     `${ind0}  ${endVar} = ${scanV}`,
@@ -4327,8 +4499,19 @@ function emitDispatch(p, ctx, pos) {
       return emitLazy(p, def, ctx, pos);
     case "trivia":
       return emit(def.parser, ctx, pos);
-    case "label":
-      return emit(def.parser, ctx, pos);
+    case "token":
+      return emitToken(def, ctx, pos);
+    case "label": {
+      const inner = emitFallible(def.parser, ctx, pos);
+      return {
+        stmts: [
+          ...inner.stmts,
+          ...emitIfFail(ctx, `!${inner.okVar}`, failBody(ctx, JSON.stringify(def.label), "_ctx._fe"))
+        ],
+        valueVar: inner.valVar,
+        endVar: inner.endVar
+      };
+    }
     case "grammar": {
       const savedTrivia = ctx.activeTrivia;
       const savedKindLabels = ctx.triviaKindLabels;
@@ -4428,6 +4611,7 @@ function hasNodeDef(p, seen = /* @__PURE__ */ new Set()) {
     }
     case "grammar":
     case "trivia":
+    case "token":
     case "label":
     case "optional":
     case "many":
@@ -4463,6 +4647,7 @@ function childrenOf(def) {
     case "optional":
     case "transform":
     case "trivia":
+    case "token":
     case "label":
     case "grammar":
     case "not":
@@ -5046,7 +5231,7 @@ function balanced(open, close, options = {}) {
   const content = bounded ? regex(new RegExp(`[^${cls}]+`)) : regex(new RegExp(`[^${escapeClassChar(open)}${escapeClassChar(close)}]`));
   const inner = many(choice(self, ...skips, content));
   const combi = transform(
-    sequence(literal(open), inner, expect(literal(close), close)),
+    sequence(literal(open), inner, expect(literal(close))),
     // parts: strings (content/self) or arrays (a sequence-shaped skip) or a
     // ParseError (recovered close). `c` is the close string or a ParseError.
     ([o, parts, c]) => o + parts.map((p) => typeof p === "string" ? p : Array.isArray(p) ? p.join("") : "").join("") + (typeof c === "string" ? c : "")
@@ -5083,6 +5268,7 @@ function childrenOf2(def) {
     case "optional":
     case "transform":
     case "trivia":
+    case "token":
     case "label":
     case "not":
     case "node":
@@ -5164,6 +5350,7 @@ function evalRuleMapIR(ir) {
     "node",
     "parser",
     "scanTo",
+    "token",
     "transform",
     "skip",
     "trivia",
@@ -5189,6 +5376,7 @@ function evalRuleMapIR(ir) {
     node,
     parser,
     scanTo,
+    token,
     transform,
     skip,
     trivia,
@@ -5322,6 +5510,8 @@ ${body}
         return `not(${kid(def.parser)})`;
       case "trivia":
         return `trivia(${kid(def.parser)})`;
+      case "token":
+        return `token(${kid(def.parser)})`;
       case "label":
         return `label(${JSON.stringify(def.label)}, ${kid(def.parser)})`;
       case "expect":
@@ -5444,6 +5634,7 @@ var SUPPORTED = {
   trivia: (...a) => trivia(a[0]),
   label: (...a) => label(a[0], a[1]),
   noTrivia: (...a) => noTrivia(a[0]),
+  token: (...a) => token(a[0]),
   expect: (...a) => expect(a[0], a[1])
 };
 function isScopeEntry(v2) {