parseman 0.46.0 → 0.47.0

package/src/index.ts

@@ -0,0 +1,189 @@
+export type { Combinator, CompiledParser, ParseResult, ParseOk, ParseFail, ParseContext, ParseError, Span, ParserMeta, FirstSet, CharRange, ParserDef, ChoiceStrategy, AutoNotCheck, GatedArm, DispatchCase, DispatchMatcherCase, BuildHost, CstCollapsePredicate, FieldCapture, FieldMap } from './types.ts'
+
+export { literal } from './combinators/literal.ts'
+export type { LiteralOptions } from './combinators/literal.ts'
+
+// `regex()` derives its own first-set: the analyzer (`./regex/first-set.ts`) is
+// imported directly by `./combinators/regex.ts`, NOT registered here. It used to
+// be wired in from this entry, which made `regex()`'s first-set depend on whether
+// the library entry was in the bundle's module graph — and `parseman/table` is a
+// separate graph, so its `regex()` fell back to the permissive `any()` set and
+// `classifiedTrivia()` rejected every table-lowered grammar. See the comment on
+// the import in `./combinators/regex.ts`.
+export { regex } from './combinators/regex.ts'
+export { keywords, word, makeWord } from './combinators/keywords.ts'
+export type { KeywordsOptions, WordOptions } from './combinators/keywords.ts'
+
+export { sequence } from './combinators/sequence.ts'
+export { choice } from './combinators/choice.ts'
+export { dispatch, endsWith, makeWhen, matches, otherwise, routed, startsWith, when } from './combinators/dispatch.ts'
+export type { DispatchArm, DispatchOtherwise, DispatchStringMatcher, DispatchWhen, DispatchWhenFactory, DispatchWhenOptions, DispatchWhenMatcher } from './combinators/dispatch.ts'
+export { attempt } from './combinators/attempt.ts'
+export { many, oneOrMore, optional, sepBy, oneOrMoreSep, keepSeparator } from './combinators/repeat.ts'
+export type { RepeatOptions, SepByOptions, TrailingSeparator, KeptSeparator } from './combinators/repeat.ts'
+export { rules } from './combinators/parser.ts'
+export { ref } from './combinators/ref.ts'
+export { not } from './combinators/not.ts'
+export { peek } from './combinators/peek.ts'
+export { node } from './combinators/node.ts'
+export type { BuildNode, NodeCombinator, NodeOptions, NodeProjectOptions } from './combinators/node.ts'
+// lazy() is intentionally NOT exported.
+export { transform, trivia, classifiedTrivia, label, field } from './combinators/map.ts'
+export { parse, parser, noTrivia } from './combinators/grammar.ts'
+export type { ParseOptions, ParserOptions, ParsemanParser } from './combinators/grammar.ts'
+export { token, leaf } from './combinators/token.ts'
+
+export { compile } from './table/compile.ts'
+/*
+ * `CompiledParser` is exported above, from `./types.ts` — it is the contract
+ * `compile()` answers to, so it lives with the library's types.
+ *
+ * TWO NAMES USED TO SIT HERE and no longer do. Neither was ever documented and
+ * nothing outside the source lowering referenced either; they were in the barrel
+ * because the barrel re-exported the engine, not because anything asked for them.
+ *
+ *   `LinkablePieces`    — the SOURCE-LOWERING linkable IR. Every field of it is
+ *                         source text or an input to resolving source text (see
+ *                         `compile-linkable-table.ts`), so it has no meaning for a
+ *                         table artifact and dies with the source lowering. The
+ *                         public composable-piece type is `LinkableTable`, below.
+ *   `DuplicationOption` — an option bag for the source lowering's duplication
+ *                         diagnostic, used nowhere else. The public diagnostic
+ *                         surface is `analyzeDuplication` /
+ *                         `AnalyzeDuplicationOptions`, exported further down.
+ *
+ * `HostMode` stays: it names the `hostMode` option on `rules()` and `compile()`,
+ * which is public. It now comes from the artifact↔host contract module that has
+ * owned the definition all along.
+ */
+export type { HostMode } from './cst/host-mode.ts'
+/**
+ * `linkable()` produces TABLES too — the table counterpart of `compileLinkable`.
+ * A piece is a table when it is self-contained, and always carries its IR, which
+ * is what makes table-to-table composition a rule-map merge and one encode rather
+ * than a merge of two already-encoded programs.
+ */
+export { compileLinkableTable, type LinkableTable, type LinkableTableOptions } from './compiler/compile-linkable-table.ts'
+export type { DuplicationOption } from './table/duplication-hook.ts'
+
+// THE diagnostic entry point. `compile()` never reports anything: diagnostics are a
+// deliberate act, not a side effect of producing an artifact. `diagnoseGrammar` accepts
+// any grammar shape, returns a deterministic machine-readable object that fails CLOSED,
+// and `formatGrammarDiagnosis` renders it for a human.
+//
+//   const d = diagnoseGrammar(myGrammar)
+//   if (!d.ok) { console.error(formatGrammarDiagnosis(d).join('\n')); process.exit(1) }
+//
+// `examinedNothing(d)` separates "measured, and it is bad" from "could not measure":
+// `ok` is false for both, and a gate that treats them the same reports a confident
+// finding count over a grammar it never opened.
+//
+// The `analyze*` functions below remain the lower-level surface it is built on.
+export { diagnoseGrammar, examinedNothing, formatGrammarDiagnosis } from './analysis/diagnose.ts'
+export type {
+  GrammarDiagnosis, DiagnosisFinding, DiagnosisCode, DiagnosisSeverity,
+  DiagnosableGrammar, DiagnoseOptions,
+} from './analysis/diagnose.ts'
+
+export { analyzeGating, analyzeGatingRules, formatGatingWarnings, firstSetToString } from './analysis/gating.ts'
+export type {
+  GatingReport, ChoiceGating, AnyArm, Overlap, AntiPattern, Unanalysable,
+  FirstSetCause, ChoiceStrategyTag, AnalyzeGatingOptions,
+} from './analysis/gating.ts'
+// Analyze a WHOLE grammar, including a `compose()` result — whose fused map holds
+// rule FUNCTIONS, not combinators, and so cannot be walked by `analyzeGatingRules`
+// directly. This is the entry point a composed grammar's author wants.
+export { analyzeGrammarGating } from './analysis/grammar.ts'
+
+// Choice-cost diagnostics: the static shared-prefix inventory, the interpreted
+// corpus wasted-work profile, and the gate policy over them. Quiet by default —
+// nothing here runs or prints unless it is called.
+export { analyzeChoiceInventory, profileWastedWork, choiceSiteKey, armLabel } from './analysis/choice-cost.ts'
+export type {
+  ChoiceSite, ChoiceInventoryEntry, ChoiceInventoryReport, PrefixGroup,
+  ArmDeclineReason, SiteDeclineReason, WastedWorkCorpusEntry, ProfileWastedWorkOptions,
+  WastedWorkArm, WastedWorkSite, WastedWorkReport,
+} from './analysis/choice-cost.ts'
+export { renderChoiceInventory, renderWastedWork, leftFactorPreview } from './analysis/choice-cost-render.ts'
+export type { RenderOptions } from './analysis/choice-cost-render.ts'
+export { checkWastedWork, buildWastedWorkBaseline } from './analysis/choice-cost-gate.ts'
+export type { WastedWorkBaseline, WastedWorkPolicy, GateBreach, GateVerdict } from './analysis/choice-cost-gate.ts'
+export type { AnalysableGrammar } from './analysis/grammar.ts'
+
+export {
+  analyzeDuplication, analyzeDuplicationRules, formatDuplicationFindings,
+  duplicationFindingCount, siteToString, alternationGroups, keywordRegexShape,
+  extractCharClasses, charClassMembers, keywordAlternationHazards,
+} from './analysis/duplication.ts'
+export type {
+  DuplicationReport, AnalyzeDuplicationOptions, DuplicationWarnLevel, Site,
+  DuplicateFinding, NearDuplicateFinding, RegexFragmentFinding,
+  RegexClassFinding, RegexClassVariant,
+  ArmOverlapFinding, RewriteFinding, RewriteKind, SepByVerdict, KeywordRegexFinding, DivergentNodeFinding,
+  StructureLossFinding,
+} from './analysis/duplication.ts'
+// `composeLeaf()` is terminal by design; ordinary reusable grammar composition is
+// `compose()`. There is no à-la-carte rule selection: compose small pieces instead.
+//
+// `fuseInterpreted()` / `isInterpretedFuse()` are deliberately NOT re-exported. The
+// interpreted fuse is a DIAGNOSTIC engine — it is how the differential harnesses and
+// bench legs run a grammar without reaching codegen — not a supported way to ship a
+// parser. Publishing it advertised a second engine with the same grammar and different
+// runtime characteristics, and `isInterpretedFuse` existed as an escape hatch that not
+// even Parseman called. Both stay internal (./compiler/linker.ts).
+//
+// Withdrawing the discriminator is only safe because `composeLeaf()` no longer LIES
+// about which engine it returned: it is typed `Record<string, Runnable>`, true on both
+// paths, and `Runnable` is what `run()`/`parseDoc()` take. Were that type ever narrowed
+// back to `FusedRule`, `isInterpretedFuse` would have to be re-exported in the same
+// change — a dual shape with no way to detect it is the worse of the two.
+export { compose, composeLeaf, cstBuildHost } from './compiler/linker.ts'
+export type { CstBuildHostOptions, FusedRule } from './compiler/linker.ts'
+
+export { buildLineIndex, createLineIndex, recordLineRange, normalizeLineIndex, offsetToLineCol, annotateSpan, annotateTreeSpans } from './line-index.ts'
+export type { LineIndex } from './line-index.ts'
+
+export { gate } from './combinators/gate.ts'
+export { adjacent, notAdjacent } from './combinators/adjacency.ts'
+export { withCtx } from './combinators/withCtx.ts'
+export { isParseError } from './combinators/expect.ts'
+export { expect } from './combinators/expect.ts'
+export { completionsAt } from './combinators/completions.ts'
+export { scanTo, balanced } from './combinators/scanTo.ts'
+export type { ScanToOptions } from './combinators/scanTo.ts'
+
+export type { CSTNode, CSTLeaf, CSTError, CSTTrivia, CSTChild, CSTRawChild, NodeLike } from './cst/types.ts'
+
+export { parseDoc } from './functional/doc.ts'
+export type { ParseDoc, ParseDocOptions, Registry, RuleFn } from './functional/doc.ts'
+/* THE PUBLIC PARSE PATH IS THE TABLE. `functional/run.ts` is the driver and
+ * still walks a combinator directly when handed one — that branch is the
+ * interpreter, and it stays reachable from `test/` and the identity sweep so a
+ * table/codegen divergence has something to be bisected against. It is not what
+ * the main entry hands out. See `functional/run-tabled.ts` for why the encode
+ * cannot live inside the driver (`parseman/run`'s closure) and why the cache is
+ * load-bearing rather than an optimisation. */
+export { run } from './functional/run-tabled.ts'
+export type { RootTriviaCapture, RunResult, RunOptions, Runnable } from './functional/run.ts'
+export { GRAMMAR_COVERAGE_DEFINITIONS, compiledGrammarCoverageDefinitions, createGrammarCoverageCollector, createGrammarInstrumentationContext, createGrammarTraceSink, grammarCoverageDefinitions, composedGrammarCoverageDefinitions, runWithGrammarCoverage } from './coverage.ts'
+export type { GrammarCoverageCollector, GrammarCoverageDefinition, GrammarCoverageSnapshot, GrammarInstrumentationContext, GrammarTraceEvent, GrammarTracePhase, GrammarTraceSink, GrammarTraceSnapshot } from './coverage.ts'
+export { buildTriviaIndex } from './cst/trivia-index.ts'
+export type { TriviaIndex, TriviaToken, TriviaIndexOptions } from './cst/trivia-index.ts'
+export { createVisitor } from './cst/walk.ts'
+export type { Walkable, VisitorHandler, VisitorSpec } from './cst/walk.ts'
+export type { GrammarReflection, GrammarNodeReflection, GrammarWithReflection } from './cst/reflection.ts'
+export { buildRootTriviaIndex, triviaEntries } from './cst/trivia-entries.ts'
+export type { RootTriviaGap, RootTriviaIndex, TriviaEntriesView } from './cst/trivia-entries.ts'
+export { triviaKindMask } from './cst/trivia-kinds.ts'
+
+export {
+  relativize,
+  absolutize,
+  absoluteSpanAt,
+  shiftAbsolute,
+  applyEdit,
+  relativizeCST,
+  absolutizeCST,
+  absoluteSpanCST,
+} from './cst/relative-spans.ts'
+export type { AbsNode, RelNode } from './cst/relative-spans.ts'

package/src/language-service/index.ts

@@ -0,0 +1,221 @@
+/**
+ * `parseman/language-service` — the external, grammar-agnostic IDE layer.
+ *
+ * A grammar is pure structure (it carries no recovery/completion/lint config).
+ * `languageService(grammar, config)` layers editor behaviour ON TOP, keyed by rule
+ * name = node type, without ever modifying the grammar:
+ *
+ *   - `parse(src, { tolerant })` → CST + `ParseError[]` (recovers on the compiled
+ *     fast path when `grammar` is a `compile(g, { recovery: true })` grammar).
+ *   - `diagnostics(src)` → structural parse errors + your per-node-type lint rules.
+ *   - `completionsAt(src, offset)` → the grammar's expected-token set at the cursor,
+ *     mapped through your per-rule semantic completion handlers.
+ *
+ * The grammar can be the interpreter combinator OR a compiled grammar — the domain
+ * knowledge (what to suggest after `color:`, which nodes to lint) lives here, in the
+ * consumer's config, never in the grammar.
+ */
+import type { Combinator, ParseContext, ParseResult, Span, ParseError } from '../types.ts'
+import { run } from '../functional/run.ts'
+import { completionsAt as coreCompletionsAt } from '../combinators/completions.ts'
+import { cstBuildHost } from '../compiler/linker.ts'
+import type { Walkable } from '../cst/walk.ts'
+import { parseDoc, type Registry } from '../functional/doc.ts'
+import { absolutizeCST } from '../cst/relative-spans.ts'
+import type { NodeLike } from '../cst/types.ts'
+
+export type Severity = 'error' | 'warning' | 'info'
+export type Diagnostic = { severity: Severity; message: string; span: Span }
+export type CompletionItem = { label: string; detail?: string }
+
+/** A CST node/leaf as a diagnostics handler sees it: the walk shape plus span/value. */
+export type LsNode = Walkable & { readonly span: Span; readonly value?: string }
+
+/** Context handed to a completion handler: which rule the cursor sits in and the
+ * grammar's raw expected-token labels there. */
+export type CompletionContext = { rule: string | null; expected: string[]; offset: number }
+
+export type LanguageServiceConfig = {
+  /** Lint/diagnostic rules keyed by node type (rule name). Return zero or more
+   * Diagnostics for each matching node. */
+  diagnostics?: Record<string, (node: LsNode) => Diagnostic | Diagnostic[] | null | undefined | void>
+  /** Semantic completion handlers keyed by the rule the cursor is in. Turn the
+   * grammar's structural expectation into domain suggestions. */
+  complete?: Record<string, (ctx: CompletionContext) => CompletionItem[]>
+}
+
+/** Grammar the service drives: an interpreter combinator or a compiled entry fn. */
+type Grammar = Combinator<unknown> | ((input: string, pos: number, ctx: ParseContext) => ParseResult<unknown>)
+
+/**
+ * What you hand `languageService`:
+ * - a single entry (combinator or `compile()`d fn) — stateless methods only, or
+ * - `{ rules, root }` (a `rules()` registry + its root rule name) — also unlocks
+ *   `openDocument`, the live incremental editor document.
+ */
+export type GrammarInput = Grammar | { rules: Registry<NodeLike>; root: string }
+
+/**
+ * A live incremental document — the fused editor backend. Holds a tolerant
+ * `parseDoc` whose tree survives broken input (recovery) and re-parses only the
+ * edited span on `edit()`. Diagnostics walk the maintained tree (recovered errors
+ * ride inside it as `parseError` nodes), so they stay complete across edits without
+ * re-parsing the whole document.
+ */
+export type LsDocument = {
+  /** The current CST with ABSOLUTE spans (recovered errors embedded as parseError nodes). */
+  readonly tree: LsNode | null
+  /** Apply an editor change (byte offsets in the CURRENT text); returns the next document. */
+  edit(from: number, to: number, replacement: string): LsDocument
+  /** Structural parse errors (from the tree) + your per-node-type lint rules. */
+  diagnostics(): Diagnostic[]
+  /** Completions at a cursor offset in the current text. */
+  completionsAt(offset: number): CompletionItem[]
+}
+
+export type LanguageService = {
+  parse(src: string, opts?: { tolerant?: boolean }): ParseResult<unknown> & { errors: ParseError[] }
+  diagnostics(src: string): Diagnostic[]
+  completionsAt(src: string, offset: number): CompletionItem[]
+  /** Open a live incremental document. Requires `languageService({ rules, root }, …)`. */
+  openDocument(src: string): LsDocument
+}
+
+const errorMessage = (expected: readonly string[]): string =>
+  expected.length ? `Unexpected input; expected ${expected.join(' or ')}` : 'Unexpected input'
+
+function visitTree(root: LsNode, enter: (node: LsNode) => void): void {
+  const go = (node: LsNode): void => {
+    enter(node)
+    const children = node.children as ReadonlyArray<LsNode> | undefined
+    if (Array.isArray(children)) for (const child of children) go(child)
+  }
+  go(root)
+}
+
+/**
+ * Merge every syntax-error source into one deduped diagnostic list:
+ *   (a) recovered `parseError` nodes embedded in the tree + per-node lint rules
+ *       (via {@link diagnoseTree}),
+ *   (b) residual FLAT recovery/failure errors from the run/doc (present when the
+ *       parse produced no tree, or as the flat mirror of embedded errors), and
+ *   (c) a sentinel over any trailing `unconsumedFrom..end` junk the grammar left.
+ * Deduped by span+message so an embedded error and its flat twin collapse to one,
+ * while distinct diagnostics at the same offset survive. With the reporting fixes
+ * (expect() embeds, speculative rollback is clean) (a) is already complete; the
+ * merge is what guarantees missing-closers and trailing junk are never invisible.
+ */
+function mergeDiagnostics(
+  tree: LsNode | null,
+  flat: ReadonlyArray<{ expected: readonly string[]; span: Span }>,
+  unconsumedFrom: number | null,
+  inputLength: number,
+  config: LanguageServiceConfig,
+): Diagnostic[] {
+  const out: Diagnostic[] = []
+  const seen = new Set<string>()
+  const add = (d: Diagnostic): void => {
+    const key = `${d.severity}|${d.span.start}|${d.span.end}|${d.message}`
+    if (seen.has(key)) return
+    seen.add(key)
+    out.push(d)
+  }
+  if (tree) for (const d of diagnoseTree(tree, config)) add(d)
+  for (const e of flat) add({ severity: 'error', message: errorMessage(e.expected), span: e.span })
+  if (unconsumedFrom !== null) {
+    add({ severity: 'error', message: 'Unexpected input', span: { start: unconsumedFrom, end: inputLength } })
+  }
+  return out
+}
+
+/** One walk over a CST: recovered `parseError` nodes → error diagnostics, and each
+ * node dispatched to its `config.diagnostics[type]` lint handler. Errors live in the
+ * tree, so this is the single source for both — no separate flat channel to merge. */
+function diagnoseTree(root: LsNode, config: LanguageServiceConfig): Diagnostic[] {
+  const out: Diagnostic[] = []
+  const rules = config.diagnostics
+  visitTree(root, node => {
+    if (node._tag === 'parseError') {
+      out.push({ severity: 'error', message: errorMessage((node as { expected?: string[] }).expected ?? []), span: node.span })
+      return
+    }
+    const h = rules && node.type !== undefined ? rules[node.type] : undefined
+    if (h) {
+      const d = h(node)
+      if (d) out.push(...(Array.isArray(d) ? d : [d]))
+    }
+  })
+  return out
+}
+
+export function languageService(grammar: GrammarInput, config: LanguageServiceConfig = {}): LanguageService {
+  const build = cstBuildHost()
+  const asReg = typeof grammar === 'object' && grammar !== null && 'rules' in grammar
+  const registry = asReg ? grammar.rules : undefined
+  const root = asReg ? grammar.root : undefined
+  const entry: Grammar = asReg ? (registry![root!] as Grammar) : grammar
+
+  // The grammar's ambient trivia (interpreter combinators carry it in _meta) —
+  // handed to run() so it skips a trailing whitespace/comment tail before
+  // reporting unconsumedFrom, matching parseDoc/run semantics.
+  const trivia = typeof entry !== 'function' ? (entry as Combinator<unknown>)._meta.grammarTrivia : undefined
+  const parseTolerant = (src: string) =>
+    run(entry as never, src, { tolerant: true, build, ...(trivia !== undefined ? { trivia } : {}) })
+
+  const complete = (input: string, offset: number, treeForCursor: unknown): CompletionItem[] => {
+    const expected = coreCompletionsAt(entry as never, input, offset, { tolerant: true })
+    const rule = ruleAtCursor(treeForCursor, offset)
+    const handler = rule !== null ? config.complete?.[rule] : undefined
+    if (handler) return handler({ rule, expected, offset })
+    return expected.map(label => ({ label }))
+  }
+
+  return {
+    parse(src, opts) {
+      return run(entry as never, src, { tolerant: opts?.tolerant ?? true, build }) as ParseResult<unknown> & { errors: ParseError[] }
+    },
+
+    diagnostics(src) {
+      const r = parseTolerant(src)
+      const tree = r.value && typeof r.value === 'object' ? (r.value as LsNode) : null
+      return mergeDiagnostics(tree, r.errors, r.unconsumedFrom, src.length, config)
+    },
+
+    completionsAt(src, offset) {
+      return complete(src, offset, parseTolerant(src.slice(0, offset)).value)
+    },
+
+    openDocument(src) {
+      if (!registry || root === undefined) {
+        throw new Error('openDocument requires languageService({ rules, root }, config) — a rules() registry, not a single entry')
+      }
+      const host = cstBuildHost()
+      const make = (doc: ReturnType<typeof parseDoc<NodeLike>>): LsDocument => {
+        // Absolutize once per document version (memoized in this closure).
+        const abs = doc.tree ? (absolutizeCST(doc.tree as never) as unknown as LsNode) : null
+        return {
+          get tree() { return abs },
+          edit: (from, to, replacement) => make(doc.edit(from, to, replacement)),
+          // Same unified merge as the stateless path: embedded parseError nodes +
+          // lint, the flat recovery errors the doc collected, and a trailing-junk
+          // sentinel — so a live editor document surfaces missing-closers and junk.
+          diagnostics: () => mergeDiagnostics(abs, doc.errors, doc.unconsumedFrom, doc.input.length, config),
+          completionsAt: (offset) => complete(doc.input, offset, abs),
+        }
+      }
+      return make(parseDoc<NodeLike>(registry, root, src, { tolerant: true, structuralReuse: true, build: host }))
+    },
+  }
+}
+
+/** Deepest node whose span brackets the cursor — "which rule is the cursor in".
+ * The grammar carries no cursor knowledge, so we reconstruct it from the CST. */
+function ruleAtCursor(root: unknown, offset: number): string | null {
+  if (!root || typeof root !== 'object') return null
+  let best: string | null = null
+  visitTree(root as LsNode, node => {
+    const s = node.span
+    if (s && s.start <= offset && offset <= s.end && node.type !== undefined) best = node.type
+  })
+  return best
+}

package/src/line-index.ts

@@ -0,0 +1,131 @@
+import type { Span } from './types.ts'
+
+/**
+ * Precomputed index of newline positions for O(log n) offset->line/col lookup.
+ * lineStarts[i] is the byte offset of the first character on line i+1.
+ */
+export type LineIndex = { lineStarts: number[] }
+type LineTrackingContext = {
+  trackLines?: boolean | undefined
+  _lineIndex?: LineIndex | undefined
+  _lineStarts?: number[] | undefined
+  _lineScannedTo?: number | undefined
+}
+
+export function createLineIndex(): LineIndex {
+  return { lineStarts: [0] }
+}
+
+export function buildLineIndex(input: string): LineIndex {
+  const lineStarts = [0]
+  for (let i = 0; i < input.length; i++) {
+    if (input.charCodeAt(i) === 10) lineStarts.push(i + 1)
+  }
+  return { lineStarts }
+}
+
+/**
+ * Record newline offsets for a matched range. Intended for optional parsers that
+ * collect line starts while they consume terminals. The collector is deliberately
+ * append-only: speculative parses never roll it back. Normalize once at the
+ * driver boundary before doing offset->line/column lookup.
+ */
+export function recordLineRange(index: LineIndex, input: string, start: number, end: number): void {
+  const lineStarts = index.lineStarts
+  for (let i = start; i < end; i++) {
+    if (input.charCodeAt(i) === 10) lineStarts.push(i + 1)
+  }
+}
+
+/**
+ * Records newlines from the context high-water mark through `end`. The start
+ * argument is only a caller hint: append-only tracking uses `_lineScannedTo` so
+ * overlapping calls can backfill gaps without double-scanning committed ranges.
+ */
+export function recordLineRangeFromContext(
+  ctx: LineTrackingContext,
+  input: string,
+  _startHint: number,
+  end: number,
+): void {
+  if (!ctx.trackLines) return
+  const from = ctx._lineScannedTo ?? 0
+  if (end <= from) return
+  const index = ctx._lineIndex ?? (ctx._lineStarts ? { lineStarts: ctx._lineStarts } : undefined)
+  if (!index) return
+  recordLineRange(index, input, from, end)
+  ctx._lineScannedTo = end
+}
+
+export function normalizeLineIndex(index: LineIndex): LineIndex {
+  const starts = index.lineStarts
+  starts.sort((a, b) => a - b)
+  let write = 0
+  for (let read = 0; read < starts.length; read++) {
+    const value = starts[read]!
+    if (write === 0 || starts[write - 1] !== value) starts[write++] = value
+  }
+  starts.length = write
+  if (starts.length === 0 || starts[0] !== 0) starts.unshift(0)
+  return index
+}
+
+export function offsetToLineCol(
+  index: LineIndex,
+  offset: number
+): { line: number; col: number } {
+  const { lineStarts } = index
+  let lo = 0
+  let hi = lineStarts.length - 1
+  while (lo < hi) {
+    const mid = (lo + hi + 1) >> 1
+    if (lineStarts[mid]! <= offset) lo = mid
+    else hi = mid - 1
+  }
+  return { line: lo + 1, col: offset - lineStarts[lo]! + 1 }
+}
+
+/** Fills startLine/startColumn/endLine/endColumn on a span in-place. */
+export function annotateSpan(span: Span, index: LineIndex): Span {
+  const s = offsetToLineCol(index, span.start)
+  const e = offsetToLineCol(index, span.end)
+  return {
+    ...span,
+    startLine: s.line,
+    startColumn: s.col,
+    endLine: e.line,
+    endColumn: e.col,
+  }
+}
+
+export function annotateSpanFromLineContext(
+  span: Span,
+  ctx: LineTrackingContext,
+): Span {
+  if (!ctx.trackLines) return span
+  const index = ctx._lineIndex ?? (ctx._lineStarts ? { lineStarts: ctx._lineStarts } : undefined)
+  return index ? annotateSpan(span, index) : span
+}
+
+function isSpan(value: unknown): value is Span {
+  return !!value
+    && typeof value === 'object'
+    && typeof (value as { start?: unknown }).start === 'number'
+    && typeof (value as { end?: unknown }).end === 'number'
+}
+
+/** Annotate every span-shaped object reachable from a parse result value. */
+export function annotateTreeSpans<T>(value: T, index: LineIndex, seen: WeakSet<object> = new WeakSet()): T {
+  if (!value || typeof value !== 'object') return value
+  if (seen.has(value)) return value
+  seen.add(value)
+
+  const obj = value as Record<string, unknown>
+  if (isSpan(obj.span)) obj.span = annotateSpan(obj.span, index)
+
+  for (const key of Object.keys(obj)) {
+    const child = obj[key]
+    if (child && typeof child === 'object') annotateTreeSpans(child, index, seen)
+  }
+  return value
+}