parseman 0.40.0 → 0.42.0

package/dist/spec/index.cjs

@@ -354,9 +354,11 @@ function markUnusedValues(root) {
         visit(def.parser, consumed);
         visit(def.sentinel, false);
         return;
+      case "token":
+        visit(def.parser, false);
+        return;
       case "label":
       case "trivia":
-      case "token":
       case "attempt":
       case "withCtx":
       case "expect":
@@ -475,6 +477,65 @@ function rules(a, b) {
   return cache;
 }
 
+// src/line-index.ts
+function createLineIndex() {
+  return { lineStarts: [0] };
+}
+function recordLineRange(index, input, start, end) {
+  const lineStarts = index.lineStarts;
+  for (let i = start; i < end; i++) {
+    if (input.charCodeAt(i) === 10) lineStarts.push(i + 1);
+  }
+}
+function recordLineRangeFromContext(ctx, input, _startHint, end) {
+  if (!ctx.trackLines) return;
+  const from = ctx._lineScannedTo ?? 0;
+  if (end <= from) return;
+  const index = ctx._lineIndex ?? (ctx._lineStarts ? { lineStarts: ctx._lineStarts } : void 0);
+  if (!index) return;
+  recordLineRange(index, input, from, end);
+  ctx._lineScannedTo = end;
+}
+function normalizeLineIndex(index) {
+  const starts = index.lineStarts;
+  starts.sort((a, b) => a - b);
+  let write = 0;
+  for (let read = 0; read < starts.length; read++) {
+    const value = starts[read];
+    if (write === 0 || starts[write - 1] !== value) starts[write++] = value;
+  }
+  starts.length = write;
+  if (starts.length === 0 || starts[0] !== 0) starts.unshift(0);
+  return index;
+}
+function offsetToLineCol(index, offset) {
+  const { lineStarts } = index;
+  let lo = 0;
+  let hi = lineStarts.length - 1;
+  while (lo < hi) {
+    const mid = lo + hi + 1 >> 1;
+    if (lineStarts[mid] <= offset) lo = mid;
+    else hi = mid - 1;
+  }
+  return { line: lo + 1, col: offset - lineStarts[lo] + 1 };
+}
+function annotateSpan(span, index) {
+  const s = offsetToLineCol(index, span.start);
+  const e = offsetToLineCol(index, span.end);
+  return {
+    ...span,
+    startLine: s.line,
+    startColumn: s.col,
+    endLine: e.line,
+    endColumn: e.col
+  };
+}
+function annotateSpanFromLineContext(span, ctx) {
+  if (!ctx.trackLines) return span;
+  const index = ctx._lineIndex ?? (ctx._lineStarts ? { lineStarts: ctx._lineStarts } : void 0);
+  return index ? annotateSpan(span, index) : span;
+}
+
 // src/cst/capture-buffer.ts
 var EMPTY = [];
 var EMPTY_TL = [];
@@ -516,7 +577,12 @@ function endCstNodeCapture(ctx, saved) {
   return materialized;
 }
 function pushCstLeaf(ctx, leaf2) {
-  pushCstChild(ctx, leaf2, leaf2);
+  if (!ctx.trackLines) {
+    pushCstChild(ctx, leaf2, leaf2);
+    return;
+  }
+  const annotated = leaf2 && typeof leaf2 === "object" && leaf2.span ? { ...leaf2, span: annotateSpanFromLineContext(leaf2.span, ctx) } : leaf2;
+  pushCstChild(ctx, annotated, annotated);
 }
 function pushCstChild(ctx, built, rawEntry) {
   const b = ctx._cstBuf;
@@ -706,7 +772,9 @@ function firstSetSentinel(fs) {
 function recoverScan(input, from, ctx, sync, expected) {
   let scanPos = from;
   while (scanPos < input.length && !probeAt(sync, input, scanPos, ctx)) scanPos++;
-  const error = { _tag: "parseError", span: { start: from, end: scanPos }, expected };
+  const span = { start: from, end: scanPos };
+  if (ctx.trackLines) recordLineRangeFromContext(ctx, input, from, scanPos);
+  const error = { _tag: "parseError", span: ctx.trackLines ? annotateSpanFromLineContext(span, ctx) : span, expected };
   ctx._errors?.push(error);
   return { error, end: scanPos };
 }
@@ -808,7 +876,8 @@ function expect(combinator, label2) {
     parse(input, pos, ctx) {
       const result = combinator.parse(input, pos, ctx);
       if (result.ok) return result;
-      const error = { _tag: "parseError", span: { start: pos, end: pos }, expected };
+      const span = { start: pos, end: pos };
+      const error = { _tag: "parseError", span: ctx.trackLines ? annotateSpanFromLineContext(span, ctx) : span, expected };
       ctx._errors?.push(error);
       if (ctx._tolerant) captureError(ctx, error);
       return { ok: true, value: error, span: { start: pos, end: pos } };
@@ -1148,6 +1217,7 @@ function literal(value, opts = {}) {
     canMatchNewline: value.includes("\n"),
     isTrivia: false
   };
+  const canMatchNewline = meta.canMatchNewline;
   if (caseInsensitive) {
     const upper = value.toUpperCase();
     const lower = value.toLowerCase();
@@ -1164,6 +1234,7 @@ function literal(value, opts = {}) {
     return function parse2(input, pos, ctx) {
       if (input.charCodeAt(pos) === code) {
         const span = { start: pos, end: pos + 1 };
+        if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, span.end);
         if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value, span });
         return { ok: true, value, span };
       }
@@ -1177,6 +1248,7 @@ function literal(value, opts = {}) {
     const matchedValue = caseInsensitive ? input.slice(pos, end) : value;
     if (caseInsensitive ? asciiFoldEq(matchedValue, value) : input.startsWith(value, pos)) {
       const span = { start: pos, end };
+      if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, end);
       if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value: matchedValue, span });
       return { ok: true, value: matchedValue, span };
     }
@@ -1341,6 +1413,7 @@ function regex(pattern, flags = "") {
         if (scanEnd === null) return failAt(ctx, [`/${source}/`], pos);
         const value = input.slice(pos, scanEnd);
         const span2 = { start: pos, end: scanEnd };
+        if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, scanEnd);
         if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value, span: span2 });
         return { ok: true, value, span: span2 };
       }
@@ -1350,6 +1423,7 @@ function regex(pattern, flags = "") {
         return failAt(ctx, [`/${source}/`], pos);
       }
       const span = { start: pos, end: pos + m[0].length };
+      if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, span.end);
       if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: "leaf", value: m[0], span });
       return { ok: true, value: m[0], span };
     }
@@ -1500,6 +1574,18 @@ function rollbackTrivia(ctx, mark) {
   rollbackCstCapture(ctx, { raw: mark.raw, tlog: mark.tlog, leaves: mark.leaves, fields: mark.fields, errors: mark.errors });
   if (ctx._triviaLog && ctx._triviaLog.length !== mark.log) ctx._triviaLog.length = mark.log;
 }
+function parseTriviaNoCapture(triviaP, input, cur, ctx) {
+  const probeCtx = {
+    trackLines: ctx.trackLines,
+    state: ctx.state,
+    ...ctx._lineIndex ? { _lineIndex: ctx._lineIndex } : {},
+    ...ctx._lineStarts ? { _lineStarts: ctx._lineStarts } : {},
+    ...ctx._lineScannedTo !== void 0 ? { _lineScannedTo: ctx._lineScannedTo } : {}
+  };
+  const result = triviaP.parse(input, cur, probeCtx);
+  ctx._lineScannedTo = probeCtx._lineScannedTo;
+  return result;
+}
 function saveLookaheadMark(ctx) {
   return { ...saveTriviaMark(ctx), probeBest: ctx._probe ? ctx._probe.best : null };
 }
@@ -1522,13 +1608,29 @@ function scanWithLabels(input, cur, ctx) {
 function advanceTrivia(input, cur, ctx) {
   const triviaP = ctx.trivia;
   if (!triviaP) return cur;
+  if (!ctx.trackLines) {
+    const fast2 = fastTriviaScanner(triviaP);
+    if (fast2) return fast2(input, cur);
+    if (ctx.triviaKindLabels) {
+      const scan = scanWithLabels(input, cur, ctx);
+      return scan.end;
+    }
+    const tr2 = triviaP.parse(input, cur, { trackLines: ctx.trackLines, state: ctx.state });
+    return tr2.ok && tr2.span.end > cur ? tr2.span.end : cur;
+  }
+  const trackTriviaLines = ctx.trackLines && triviaP._meta.canMatchNewline;
   const fast = fastTriviaScanner(triviaP);
-  if (fast) return fast(input, cur);
+  if (fast) {
+    const end = fast(input, cur);
+    if (trackTriviaLines) recordLineRangeFromContext(ctx, input, cur, end);
+    return end;
+  }
   if (ctx.triviaKindLabels) {
     const scan = scanWithLabels(input, cur, ctx);
+    if (trackTriviaLines) recordLineRangeFromContext(ctx, input, cur, scan.end);
     return scan.end;
   }
-  const tr = triviaP.parse(input, cur, { trackLines: ctx.trackLines, state: ctx.state });
+  const tr = parseTriviaNoCapture(triviaP, input, cur, ctx);
   return tr.ok && tr.span.end > cur ? tr.span.end : cur;
 }
 function scanTrivia(input, cur, ctx) {
@@ -1536,18 +1638,46 @@ function scanTrivia(input, cur, ctx) {
   if (!triviaP) return { end: cur, commit: NOOP_COMMIT };
   const log = ctx._triviaLog;
   const captureTl = ctx.captureTrivia && (ctx._cstBuf !== void 0 || ctx._cstTriviaLog !== void 0);
+  if (!ctx.trackLines) {
+    const fast2 = !ctx.triviaKindLabels ? fastTriviaScanner(triviaP) : null;
+    if (fast2 && log === void 0 && !captureTl) {
+      return { end: fast2(input, cur), commit: NOOP_COMMIT };
+    }
+    if (ctx.triviaKindLabels && (log !== void 0 || captureTl)) {
+      return scanWithLabels(input, cur, ctx);
+    }
+    if (log !== void 0 || captureTl) {
+      const tr3 = triviaP.parse(input, cur, {
+        trackLines: log !== void 0 ? false : ctx.trackLines,
+        state: ctx.state
+      });
+      if (!tr3.ok || tr3.span.end === cur) return { end: cur, commit: NOOP_COMMIT };
+      const end = tr3.span.end;
+      return {
+        end,
+        commit: () => {
+          pushTriviaLogEntry(ctx, cur, end);
+          if (captureTl) pushCstTriviaEntry(ctx, cur, end);
+        }
+      };
+    }
+    const tr2 = triviaP.parse(input, cur, { trackLines: ctx.trackLines, state: ctx.state });
+    return { end: tr2.ok ? tr2.span.end : cur, commit: NOOP_COMMIT };
+  }
+  const trackTriviaLines = ctx.trackLines && triviaP._meta.canMatchNewline;
   const fast = !ctx.triviaKindLabels ? fastTriviaScanner(triviaP) : null;
   if (fast && log === void 0 && !captureTl) {
-    return { end: fast(input, cur), commit: NOOP_COMMIT };
+    const end = fast(input, cur);
+    if (trackTriviaLines) recordLineRangeFromContext(ctx, input, cur, end);
+    return { end, commit: NOOP_COMMIT };
   }
   if (ctx.triviaKindLabels && (log !== void 0 || captureTl)) {
-    return scanWithLabels(input, cur, ctx);
+    const scan = scanWithLabels(input, cur, ctx);
+    if (trackTriviaLines) recordLineRangeFromContext(ctx, input, cur, scan.end);
+    return scan;
   }
   if (log !== void 0 || captureTl) {
-    const tr2 = triviaP.parse(input, cur, {
-      trackLines: log !== void 0 ? false : ctx.trackLines,
-      state: ctx.state
-    });
+    const tr2 = parseTriviaNoCapture(triviaP, input, cur, ctx);
     if (!tr2.ok || tr2.span.end === cur) return { end: cur, commit: NOOP_COMMIT };
     const end = tr2.span.end;
     return {
@@ -1558,7 +1688,7 @@ function scanTrivia(input, cur, ctx) {
       }
     };
   }
-  const tr = triviaP.parse(input, cur, { trackLines: ctx.trackLines, state: ctx.state });
+  const tr = parseTriviaNoCapture(triviaP, input, cur, ctx);
   return { end: tr.ok ? tr.span.end : cur, commit: NOOP_COMMIT };
 }
 function consumeTrivia(input, cur, ctx) {
@@ -2211,7 +2341,7 @@ function scanTo(sentinel, { skip: skip2, raw = false, orEOF = false } = {}) {
       let cur = pos;
       const skip3 = resolveScanSkip(explicitSkip, raw, ctx);
       const probeCtx = {
-        trackLines: ctx.trackLines,
+        trackLines: false,
         state: ctx.state,
         ...ctx._errors !== void 0 ? { _errors: ctx._errors } : {}
       };
@@ -2224,6 +2354,7 @@ function scanTo(sentinel, { skip: skip2, raw = false, orEOF = false } = {}) {
       while (cur < input.length) {
         const s = sentinel.parse(input, cur, probeCtx);
         if (s.ok) {
+          if (ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, cur);
           emit(cur);
           return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
         }
@@ -2239,6 +2370,7 @@ function scanTo(sentinel, { skip: skip2, raw = false, orEOF = false } = {}) {
         if (!advanced) cur++;
       }
       if (orEOF) {
+        if (ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, cur);
         emit(cur);
         return { ok: true, value: input.slice(pos, cur), span: { start: pos, end: cur } };
       }
@@ -2256,7 +2388,7 @@ var META = new Set("()[]{}*+?|^$.".split(""));
 var CAP_RECORD = [
   `  if (_cap && _e > _pos) {`,
   `    if (_ctx._triviaLog !== undefined) _ctx._triviaLog.push(_pos, _e)`,
-  `    if (_ctx._cstTriviaLog !== undefined && _ctx.captureTrivia) _ctx._cstTriviaLog.push(_pos, _e, _ctx._cstRawChildren ? _ctx._cstRawChildren.length : 0)`,
+  `    if (_cap === 1 && _ctx._cstTriviaLog !== undefined && _ctx.captureTrivia) _ctx._cstTriviaLog.push(_pos, _e, _ctx._cstRawChildren ? _ctx._cstRawChildren.length : 0)`,
   `  }`
 ].join("\n");
 
@@ -2848,49 +2980,36 @@ function node(typeOrCombinator, combinatorOrBuild, buildOrOpts, maybeOpts) {
       const st = clonesState && ctx.state !== void 0 ? Object.assign({}, ctx.state) : void 0;
       const nodeType = def.type ?? missingInferredType();
       const cstOutput = cstOutputHost(ctx.build);
+      const span = ctx.trackLines ? annotateSpanFromLineContext(r.span, ctx) : r.span;
       const hostState = clonesState ? st : ctx.state !== void 0 ? Object.assign({}, ctx.state) : void 0;
-      const built = unwrap && children.length === 1 ? unwrapChild(children[0]) : collapse && children.length === 1 ? children[0] : !build && project === void 0 && ctx.build?._parsemanCstCollapse && children.length === 1 && rawChildren.length === 1 && ctx.build._parsemanCstCollapse(nodeType, children[0], children, rawChildren) ? children[0] : project !== void 0 ? cstOutput && ctx.build ? ctx.build(nodeType, children, fields, r.span, rawChildren, triviaLog, hostState) : projectChild(children, project, nodeType) : build ? cstOutput && ctx.build ? ctx.build(nodeType, children, fields, r.span, rawChildren, triviaLog, hostState) : build(children, fields, r.span, rawChildren, triviaLog, st) : ctx.build ? ctx.build(nodeType, children, fields, r.span, rawChildren, triviaLog, st) : { _tag: "node", type: nodeType, span: { start: r.span.start, end: r.span.end }, state: st ?? null, children };
-      const rawEntry = isCstChild(built) ? built : { _tag: "leaf", value: typeof built === "string" ? built : typeof built === "object" && built !== null ? input.slice(r.span.start, r.span.end) : "", span: r.span };
+      const built = unwrap && children.length === 1 ? unwrapChild(children[0]) : collapse && children.length === 1 ? children[0] : !build && project === void 0 && ctx.build?._parsemanCstCollapse && children.length === 1 && rawChildren.length === 1 && ctx.build._parsemanCstCollapse(nodeType, children[0], children, rawChildren) ? children[0] : project !== void 0 ? cstOutput && ctx.build ? ctx.build(nodeType, children, fields, span, rawChildren, triviaLog, hostState) : projectChild(children, project, nodeType) : build ? cstOutput && ctx.build ? ctx.build(nodeType, children, fields, span, rawChildren, triviaLog, hostState) : build(children, fields, span, rawChildren, triviaLog, st) : ctx.build ? ctx.build(nodeType, children, fields, span, rawChildren, triviaLog, st) : { _tag: "node", type: nodeType, span: ctx.trackLines ? { ...span } : { start: r.span.start, end: r.span.end }, state: st ?? null, children };
+      const rawEntry = isCstChild(built) ? built : { _tag: "leaf", value: typeof built === "string" ? built : typeof built === "object" && built !== null ? input.slice(r.span.start, r.span.end) : "", span };
       if (saved.buf !== void 0 || saved.ch !== void 0) {
         pushCstChild(ctx, built, rawEntry);
       }
-      return { ok: true, value: built, span: r.span };
+      return { ok: true, value: built, span };
     }
   };
 }
 
-// src/compiler/line-index.ts
-function buildLineIndex(input) {
-  const lineStarts = [0];
-  for (let i = 0; i < input.length; i++) {
-    if (input.charCodeAt(i) === 10) lineStarts.push(i + 1);
-  }
-  return { lineStarts };
-}
-function offsetToLineCol(index, offset) {
-  const { lineStarts } = index;
-  let lo = 0;
-  let hi = lineStarts.length - 1;
-  while (lo < hi) {
-    const mid = lo + hi + 1 >> 1;
-    if (lineStarts[mid] <= offset) lo = mid;
-    else hi = mid - 1;
+// src/combinators/grammar.ts
+function annotateResultLines(result, index) {
+  const annotated = { ...result, span: annotateSpan(result.span, index) };
+  const errors = annotated.errors;
+  if (errors) {
+    ;
+    annotated.errors = errors.map((error) => ({
+      ...error,
+      span: annotateSpan(error.span, index)
+    }));
   }
-  return { line: lo + 1, col: offset - lineStarts[lo] + 1 };
+  return annotated;
 }
-function annotateSpan(span, index) {
-  const s = offsetToLineCol(index, span.start);
-  const e = offsetToLineCol(index, span.end);
-  return {
-    ...span,
-    startLine: s.line,
-    startColumn: s.col,
-    endLine: e.line,
-    endColumn: e.col
-  };
+function createParseLineContext(input, pos) {
+  const lineIndex = createLineIndex();
+  if (pos > 0) recordLineRange(lineIndex, input, 0, pos);
+  return { lineIndex, lineScannedTo: pos };
 }
-
-// src/combinators/grammar.ts
 function parser(opts, root) {
   const clearTrivia = opts.trivia === null;
   return {
@@ -2906,9 +3025,13 @@ function parser(opts, root) {
     },
     parse(input, pos, _ctx) {
       const trackLines = opts.trackLines ?? _ctx?.trackLines ?? false;
+      const lineContext = trackLines && _ctx?._lineIndex === void 0 && _ctx?._lineStarts === void 0 ? createParseLineContext(input, pos ?? 0) : void 0;
+      const lineIndex = trackLines ? _ctx?._lineIndex ?? (_ctx?._lineStarts ? { lineStarts: _ctx._lineStarts } : lineContext.lineIndex) : void 0;
       const ctx = {
         ..._ctx,
         trackLines,
+        ...lineIndex ? { _lineIndex: lineIndex } : {},
+        ...lineContext ? { _lineScannedTo: lineContext.lineScannedTo } : {},
         // trivia: null clears (contiguous terms); a Combinator sets; undefined inherits.
         ...clearTrivia ? {
           trivia: void 0,
@@ -2926,11 +3049,10 @@ function parser(opts, root) {
         ...opts.captureTriviaKinds && !clearTrivia ? { _triviaCaptureMask: triviaKindMask(opts.trivia?._meta.triviaKindLabels ?? _ctx?.triviaKindLabels, opts.captureTriviaKinds) } : {}
       };
       const result = root.parse(input, pos ?? 0, ctx);
-      if (trackLines) {
-        const idx = buildLineIndex(input);
-        return { ...result, span: annotateSpan(result.span, idx) };
+      if (trackLines && _ctx && ctx._lineScannedTo !== void 0) {
+        _ctx._lineScannedTo = Math.max(_ctx._lineScannedTo ?? 0, ctx._lineScannedTo);
       }
-      return result;
+      return lineIndex ? annotateResultLines(result, normalizeLineIndex(lineIndex)) : result;
     }
   };
 }
@@ -3218,7 +3340,7 @@ function normalizeCstCollapse(collapse) {
   return typeof collapse === "function" ? collapse : void 0;
 }
 function buildCstNode(type, children, _fields, span, _rawChildren, _triviaLog, state) {
-  return { _tag: "node", type, span: { start: span.start, end: span.end }, state: state ?? null, children: [...children] };
+  return { _tag: "node", type, span: { ...span }, state: state ?? null, children: [...children] };
 }
 function cstBuildHost(typeOrOptions, children, _fields, span, rawChildren, triviaLog, state) {
   if (typeof typeOrOptions === "string") {