parseman 0.55.4 → 0.56.0

package/src/combinators/lines.ts

@@ -0,0 +1,244 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
+import { any, matchesEmpty } from './first-set.ts'
+import { createDetachedParseContext } from '../parse-context.ts'
+import { recordLineRangeFromContext } from '../line-index.ts'
+import { failAt } from './probe.ts'
+
+/**
+ * One line of a run that `lines()` hands its body.
+ *
+ * The body parses the run's lines with their prefixes removed and joined by
+ * `\n`. Each entry says where a line's content sits in that text (`offset`) and
+ * in the run's own input (`source`), where its column count starts, and whether
+ * it joined the run through `lazy`.
+ */
+export type RunLine = {
+  /** Where the line's content starts in the body's input. */
+  readonly offset: number
+  /** Where the same content starts in the run's input. */
+  readonly source: number
+  /** Where the line ends in the run's input, before its newline. */
+  readonly end: number
+  /**
+   * The column at `source`: tab stops every four columns, counted from the
+   * start of the outermost input's line, through every enclosing run.
+   */
+  readonly column: number
+  /**
+   * Columns of the character at `source` that a prefix already took. A
+   * `columns(n)` prefix that ends inside a tab leaves the tab in the content and
+   * records the columns it took here, so the content keeps the tab's remainder.
+   */
+  readonly split: number
+  /** The line joined through `lazy`, with nothing removed. */
+  readonly lazy: boolean
+}
+
+/** The lines of the run whose body is parsing, and the run enclosing it. */
+export type LineView = {
+  readonly lines: readonly RunLine[]
+  readonly parent: LineView | undefined
+}
+
+export type LinesOptions = {
+  /** The prefix of the run's first line, when it differs from the others' (a list item's marker). */
+  first?: Combinator<unknown>
+  /**
+   * A line the prefix does not match continues the run, whole, when `lazy`
+   * matches at its start: Markdown's lazy continuation lines. Only the match
+   * counts; what `lazy` consumes is not removed.
+   */
+  lazy?: Combinator<unknown>
+}
+
+type LinesContext = ParseContext
+
+/** The index of the run line holding `offset` of the body's input, by binary search. */
+function lineIndexAt(view: LineView, offset: number): number {
+  const lines = view.lines
+  let lo = 0, hi = lines.length - 1
+  while (lo < hi) {
+    const mid = (lo + hi + 1) >> 1
+    if (lines[mid]!.offset <= offset) lo = mid
+    else hi = mid - 1
+  }
+  return lo
+}
+const lineAt = (view: LineView, offset: number): RunLine | undefined => view.lines[lineIndexAt(view, offset)]
+
+/** Map an offset in a run body's input to the offset in the run's own input. */
+export function sourceOffset(view: LineView, offset: number): number {
+  const line = lineAt(view, offset)
+  return line === undefined ? offset : line.source + (offset - line.offset)
+}
+
+/**
+ * The column at `pos` and the columns of the character there already taken,
+ * counting from the line start of the current input, or, inside a run's body,
+ * from that run's line, whose column and split the enclosing runs supply.
+ */
+function columnAt(input: string, pos: number, ctx: LinesContext): { column: number; split: number } {
+  const view = ctx.lineView
+  let start: number, column: number, split: number
+  const line = view === undefined ? undefined : lineAt(view, pos)
+  if (line !== undefined && pos >= line.offset) { start = line.offset; column = line.column; split = line.split }
+  else { start = input.lastIndexOf('\n', pos - 1) + 1; column = 0; split = 0 }
+  for (let i = start; i < pos; i++) {
+    if (input.charCodeAt(i) === 0x09) {
+      // A tab reaches the next stop; one the prefix split has fewer columns left.
+      const width = 4 - (column - (i === start ? split : 0)) % 4 - (i === start ? split : 0)
+      column += width
+    } else column++
+    split = 0
+  }
+  return { column, split: pos === start ? split : 0 }
+}
+
+/**
+ * `n` columns of indentation: spaces and tabs, a tab reaching the next stop of
+ * four columns. When the `n`th column falls inside a tab, the match stops before
+ * that tab, and a `lines()` prefix ending there leaves the tab's remaining
+ * columns in the content (`RunLine.split`). Fails when the indentation is
+ * narrower than `n`.
+ */
+export function columns(n: number): Combinator<string> {
+  if (!Number.isInteger(n) || n < 0) throw new TypeError(`columns(n) needs a non-negative integer, got ${n}`)
+  const expected = [`columns(${n})`]
+  const meta: ParserMeta = {
+    firstSet: n === 0 ? any() : { kind: 'ranges', ranges: [{ lo: 0x09, hi: 0x09 }, { lo: 0x20, hi: 0x20 }] },
+    canMatchNewline: false,
+    isTrivia: false,
+  }
+  return {
+    _tag: 'columns',
+    _meta: meta,
+    _def: { tag: 'columns', n },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
+      const c = ctx as LinesContext
+      let { column, split } = columnAt(input, pos, c)
+      const goal = column + n
+      let at = pos
+      while (column < goal) {
+        const code = input.charCodeAt(at)
+        if (code === 0x20) { column++; split = 0; at++; continue }
+        if (code !== 0x09) return { ok: false, expected, span: { start: at, end: at } }
+        const width = 4 - (column - split) % 4 - split
+        if (column + width > goal) {
+          // The goal falls inside this tab: stop before it, and say how much of it is taken.
+          c._columnsSplit = { at, split: split + (goal - column) }
+          return { ok: true, value: input.slice(pos, at), span: { start: pos, end: at } }
+        }
+        column += width; split = 0; at++
+      }
+      return { ok: true, value: input.slice(pos, at), span: { start: pos, end: at } }
+    },
+  }
+}
+
+/**
+ * Parse `body` over a run of lines with a prefix removed from each.
+ *
+ * The run starts at `pos`, where `first` (default `prefix`) must match. Each
+ * following line joins while `prefix` matches at its start, or, failing that,
+ * while `lazy` does; the run ends at the first line that joins neither way or
+ * at the end of the input. A prefix never reaches past its own line.
+ *
+ * `body` parses the lines' contents joined by `\n`, starting at 0. While it
+ * runs, `ctx.lineView` describes the lines (`RunLine`), so nested `columns()`
+ * and `lines()` count tab stops from the right column. The result spans from
+ * `pos` to where the body stopped, mapped back to this input: a body that
+ * consumes a whole line ends at the next line's start, so lines it leaves, and
+ * the rest of a line where it stops partway, are the enclosing grammar's. Spans inside the body's value and its CST are offsets in the
+ * body's input; `sourceOffset(view, offset)` maps them back. Recovered errors
+ * the body records are mapped back to this input, line tracking records the
+ * run's lines in this input, and a completion probe at a cursor inside a
+ * line's content sees what the body expects there.
+ *
+ * The run is read before the body runs, so a body that leaves most of its run
+ * makes a repeated run reread the lines it left: a body reads its run to the end.
+ */
+export function lines<T>(prefix: Combinator<unknown>, body: Combinator<T>, options: LinesOptions = {}): Combinator<T> {
+  const { first, lazy } = options
+  const opening = first ?? prefix
+  const meta: ParserMeta = {
+    // An opening that matches empty starts a run at any character.
+    firstSet: matchesEmpty(opening) ? any() : opening._meta.firstSet,
+    canMatchNewline: true,
+    isTrivia: false,
+  }
+  return {
+    _tag: 'lines',
+    _meta: meta,
+    _def: { tag: 'lines', prefix, body: body as Combinator<unknown>, ...(first === undefined ? {} : { first }), ...(lazy === undefined ? {} : { lazy }) },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<T> {
+      const c = ctx as LinesContext
+      // Prefixes and lazy checks are probes: they run detached, so they leave
+      // no leaves, trivia or errors in the parse that encloses the run.
+      const probe = createDetachedParseContext(false, ctx.state) as LinesContext
+      probe.lineView = c.lineView
+      const run: RunLine[] = []
+      const parts: string[] = []
+      let offset = 0
+      for (let start = pos; start < input.length || start === pos;) {
+        let end = input.indexOf('\n', start)
+        if (end < 0) end = input.length
+        probe._columnsSplit = undefined
+        const head = (run.length === 0 ? opening : prefix).parse(input, start, probe)
+        let content: number
+        let joined: 'prefix' | 'lazy' | undefined
+        if (head.ok && head.span.end <= end) { content = head.span.end; joined = 'prefix' }
+        else if (run.length === 0) {
+          // A prefix is a probe: its failure is the run's, never a commitment.
+          return { ok: false, expected: head.ok ? ['line prefix'] : head.expected, span: { start, end: start } }
+        } else if (lazy !== undefined && lazy.parse(input, start, probe).ok) { content = start; joined = 'lazy' }
+        else break
+        const mark = probe._columnsSplit as { at: number; split: number } | undefined
+        const here = columnAt(input, content, c)
+        const split = joined === 'prefix' && mark !== undefined && mark.at === content ? mark.split : here.split
+        run.push({ offset, source: content, end, column: here.column + (split - here.split), split, lazy: joined === 'lazy' })
+        const text = input.slice(content, end)
+        const newline = end < input.length
+        parts.push(newline ? `${text}\n` : text)
+        offset += text.length + (newline ? 1 : 0)
+        start = end + 1
+        if (!newline) break
+      }
+      const view: LineView = { lines: run, parent: c.lineView }
+      // The body parses another string: what records positions in this input
+      // (the line index, the completion probe) stays out of its way. A probe
+      // whose cursor lies in a line's content gets its own, at the body's offset.
+      const saved = c.lineView, starts = c._lineStarts, lineIndex = c._lineIndex, scanned = c._lineScannedTo, probing = c._probe
+      const errors = c._errors?.length ?? 0
+      let cursor = -1
+      if (probing !== undefined) for (const line of run) {
+        if (probing.offset >= line.source && probing.offset <= line.end) { cursor = line.offset + probing.offset - line.source; break }
+      }
+      const inner: ParseContext['_probe'] = cursor < 0 ? undefined : { offset: cursor, best: null }
+      c.lineView = view; c._lineStarts = undefined; c._lineIndex = undefined; c._lineScannedTo = undefined; c._probe = inner
+      let result: ParseResult<T>
+      try { result = body.parse(parts.join(''), 0, ctx) }
+      finally { c.lineView = saved; c._lineStarts = starts; c._lineIndex = lineIndex; c._lineScannedTo = scanned; c._probe = probing }
+      // The body's deepest failure at the cursor counts where it is in this input.
+      if (inner?.best) failAt(c, inner.best.expected, sourceOffset(view, inner.best.span.start))
+      // Errors the body recovered from move to where they are in this input.
+      if (c._errors !== undefined) for (let i = errors; i < c._errors.length; i++) {
+        const error = c._errors[i] as { span: { start: number; end: number } }
+        error.span = { start: sourceOffset(view, error.span.start), end: sourceOffset(view, error.span.end) }
+      }
+      if (!result.ok) {
+        const at = sourceOffset(view, result.span.start)
+        return { ...result, span: { start: at, end: at } }
+      }
+      // The body's end maps to the end of its line: a body that consumed a
+      // whole line, newline included, ends at the next line's start.
+      const stop = result.span.end
+      const index = lineIndexAt(view, stop)
+      const line = run[index]!
+      const end = stop === line.offset && index > 0 ? run[index - 1]!.end + 1
+        : stop - line.offset > line.end - line.source ? line.end + 1
+        : line.source + (stop - line.offset)
+      recordLineRangeFromContext(c, input, pos, end)
+      return { ok: true, value: result.value, span: { start: pos, end } }
+    },
+  }
+}

package/src/combinators/trivia-skip.ts

@@ -188,15 +188,17 @@ function parseTriviaNoCapture(
  * `peek(sequence(literal('a'), literal('zzz')))` over `"ab"` offered `"zzz"` as
  * the ONLY completion, though no input can ever reach it there.
  */
-export type LookaheadMark = TriviaRollbackMark & { probeBest: ParseFail | null }
+export type LookaheadMark = TriviaRollbackMark & { probeBest: ParseFail | null; split: ParseContext['_columnsSplit'] }
 
 export function saveLookaheadMark(ctx: ParseContext): LookaheadMark {
-  return { ...saveTriviaMark(ctx), probeBest: ctx._probe ? ctx._probe.best : null }
+  return { ...saveTriviaMark(ctx), probeBest: ctx._probe ? ctx._probe.best : null, split: ctx._columnsSplit }
 }
 
 export function rollbackLookahead(ctx: ParseContext, mark: LookaheadMark): void {
   rollbackTrivia(ctx, mark)
   if (ctx._probe) ctx._probe.best = mark.probeBest
+  // A lookahead takes no columns of a tab: `lines()` reads `columns()`'s split mark.
+  ctx._columnsSplit = mark.split
 }
 
 function scanWithLabels(input: string, cur: number, ctx: ParseContext): CompactTriviaScan {

package/src/compiler/fields.ts

@@ -30,6 +30,7 @@ export function parserHasOwnFields(p: Combinator<unknown>, seen: Set<Combinator<
       || (d.recoverAt !== undefined && parserHasOwnFields(d.recoverAt, seen))
       || (d.stopAt !== undefined && parserHasOwnFields(d.stopAt, seen))
     case 'recover': return parserHasOwnFields(d.parser, seen) || parserHasOwnFields(d.sentinel, seen)
+    case 'lines': return parserHasOwnFields(d.body, seen)
     case 'routed': return d.fallback ? parserHasOwnFields(d.fallback, seen) : false
     case 'many':
     case 'oneOrMore':
@@ -98,6 +99,7 @@ function hasTriviaSite(p: Combinator<unknown>, seen: Set<Combinator<unknown>>, t
     case 'sepBy':
     case 'scanTo':
     case 'recover':
+    case 'lines':
       return true
     // Nested node manages its own trivia frame; none logs into THIS frame. The
     // root log has no frames, so it must look inside.
@@ -159,6 +161,7 @@ export function parserEnablesTriviaCapture(p: Combinator<unknown>, seen: Set<Com
       || (d.recoverAt !== undefined && parserEnablesTriviaCapture(d.recoverAt, seen))
       || (d.stopAt !== undefined && parserEnablesTriviaCapture(d.stopAt, seen))
     case 'recover': return parserEnablesTriviaCapture(d.parser, seen) || parserEnablesTriviaCapture(d.sentinel, seen)
+    case 'lines': return parserEnablesTriviaCapture(d.body, seen)
     case 'routed': return d.fallback ? parserEnablesTriviaCapture(d.fallback, seen) : false
     case 'many':
     case 'oneOrMore':

package/src/compiler/grammar-coverage-ids.ts

@@ -39,6 +39,7 @@ function children(def: ParserDef, winners?: Record<string, Combinator<unknown>>)
       ...(def.recoverAt === undefined ? [] : [def.recoverAt]),
       ...(def.stopAt === undefined ? [] : [def.stopAt]),
     ]
+    case 'lines': return [def.prefix, def.body, ...(def.first === undefined ? [] : [def.first]), ...(def.lazy === undefined ? [] : [def.lazy])]
     case 'lazy': {
       let resolved: Combinator<unknown>
       try { resolved = def.thunk() } catch (e) {

package/src/compiler/ir-serialize.ts

@@ -72,6 +72,8 @@ function childrenOf(def: ParserDef): Comb[] {
       ...(def.stopAt === undefined ? [] : [def.stopAt]),
     ]
     case 'routed':    return def.fallback ? [def.fallback] : []
+    case 'lines':     return [def.prefix, def.body, ...(def.first === undefined ? [] : [def.first]), ...(def.lazy === undefined ? [] : [def.lazy])]
+    case 'columns':
     case 'lazy':
     case 'literal':
     case 'regex':
@@ -488,6 +490,14 @@ class Serializer {
       case 'expect':    return `expect(${kid(def.parser)}${def.label !== undefined ? `, ${JSON.stringify(def.label)}` : ''})`
       case 'scanTo':
         return `scanTo(${kid(def.sentinel)}, { skip: [${def.skip.map(kid).join(', ')}]${def.raw ? ', raw: true' : ''}, orEOF: ${def.orEOF}${def.recoverAt === undefined ? '' : `, recoverAt: ${kid(def.recoverAt)}`}${def.stopAt === undefined ? '' : `, stopAt: ${kid(def.stopAt)}`} })`
+      case 'lines': {
+        const options = [
+          ...(def.first === undefined ? [] : [`first: ${kid(def.first)}`]),
+          ...(def.lazy === undefined ? [] : [`lazy: ${kid(def.lazy)}`]),
+        ]
+        return `lines(${kid(def.prefix)}, ${kid(def.body)}${options.length === 0 ? '' : `, { ${options.join(', ')} }`})`
+      }
+      case 'columns':   return `columns(${def.n})`
       case 'transform': {
         if (def.fnSrc === undefined) throw new Unserializable('transform without fnSrc')
         // `_tf` sets `_def.fnSrc` so re-lowering INLINES the callback (a plain

package/src/compiler/token-alphabet.ts

@@ -1698,6 +1698,8 @@ function normalizeLexicalBody(
     case 'adjacency': return done({ refusal: 'adjacency depends on parser state' })
     case 'recover': return done({ refusal: 'recover is effectful' })
     case 'scanTo': return done({ refusal: 'scanTo is not a lexical run' })
+    case 'lines': return done({ refusal: 'lines removes prefixes from its input' })
+    case 'columns': return done({ refusal: 'columns counts tab stops' })
     case 'sepBy': return done({ refusal: 'separated repetition is not a lexical run' })
     case 'dispatch': return done({ refusal: 'nested dispatch has semantic outcomes' })
     case 'routed': return done({ refusal: 'routed depends on an outer dispatch' })
@@ -2026,7 +2028,7 @@ function decisionLead(
       return finish({ gap: 'repeated or nullable leading language needs a range-partition proof' })
     case 'peek': return finish({ gap: 'positive lookahead needs an exact effect-free compatible view' })
     case 'not': return finish({ gap: 'negative lookahead needs a decidable complement' })
-    case 'withCtx': case 'guard': case 'adjacency': case 'recover': case 'scanTo': case 'routed': case 'unknown':
+    case 'withCtx': case 'guard': case 'adjacency': case 'recover': case 'scanTo': case 'routed': case 'unknown': case 'lines': case 'columns':
       return finish({ gap: `leading ${def.tag} language is not represented` })
     case 'literal': case 'keywords': case 'regex':
       throw new Error('parseman: decision lead terminal escaped its direct case')

package/src/coverage.ts

@@ -1,3 +1,4 @@
+import { lines } from './combinators/lines.ts'
 import { run, type RunOptions, type RunResult, type Runnable } from './functional/run.ts'
 import { GRAMMAR_COVERAGE_DEFINITIONS } from './grammar-metadata.ts'
 export { GRAMMAR_COVERAGE_DEFINITIONS } from './grammar-metadata.ts'
@@ -365,6 +366,11 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
           ...(def.recoverAt === undefined ? {} : { recoverAt: build(def.recoverAt) }),
           ...(def.stopAt === undefined ? {} : { stopAt: build(def.stopAt) }),
         })
+        case 'lines': return lines(build(def.prefix), build(def.body), {
+          ...(def.first === undefined ? {} : { first: build(def.first) }),
+          ...(def.lazy === undefined ? {} : { lazy: build(def.lazy) }),
+        })
+        case 'columns': return parser
         // Zero-width assertion with no coverage-bearing child; reuse it verbatim so
         // the rebuilt sequence still recognises the boundary marker.
         case 'adjacency': return parser

package/src/cst/reflection.ts

@@ -66,6 +66,9 @@ function childrenOf(def: ParserDef): readonly Combinator<unknown>[] {
       ]
     case 'routed':
       return def.fallback ? [def.fallback] : []
+    case 'lines':
+      return [def.prefix, def.body, ...(def.first === undefined ? [] : [def.first]), ...(def.lazy === undefined ? [] : [def.lazy])]
+    case 'columns':
     case 'literal':
     case 'regex':
     case 'keywords':

package/src/index.ts

@@ -152,6 +152,8 @@ export { isParseError } from './combinators/expect.ts'
 export { expect } from './combinators/expect.ts'
 export { completionsAt } from './combinators/completions.ts'
 export { scanTo, balanced } from './combinators/scanTo.ts'
+export { lines, columns, sourceOffset } from './combinators/lines.ts'
+export type { LinesOptions, LineView, RunLine } from './combinators/lines.ts'
 export type { ScanToOptions } from './combinators/scanTo.ts'
 
 export type { CSTNode, CSTLeaf, CSTError, CSTTrivia, CSTChild, CSTRawChild, NodeLike } from './cst/types.ts'

package/src/parse-context.ts

@@ -48,6 +48,8 @@ export function createParseContext(): ParseContext {
     _triviaCaptureMask: undefined,
     trackLines: false,
     state: undefined,
+    lineView: undefined,
+    _columnsSplit: undefined,
     build: undefined,
     _errors: undefined,
     _tolerant: undefined,

package/src/plugin/evaluator.ts

@@ -320,6 +320,10 @@ const SUPPORTED: Record<string, (...args: unknown[]) => Combinator<unknown>> = {
   field:     (...a) => parseman.field(a[0] as string, a[1] as Combinator<unknown>, a[2] as parseman.FieldOptions | undefined),
   noTrivia:  (...a) => parseman.noTrivia(a[0] as Combinator<unknown>),
   token:     (...a) => parseman.token(a[0] as Combinator<unknown>),
+  // `lines(prefix, body, { first, lazy })`: its options are combinators, which the
+  // generic argument evaluation already resolves; a missing option stays absent.
+  lines:     (...a) => parseman.lines(a[0] as Combinator<unknown>, a[1] as Combinator<unknown>, a[2] as parseman.LinesOptions | undefined),
+  columns:   (...a) => parseman.columns(a[0] as number),
   // `routed(fallback)` — the fallback must be forwarded. A zero-arg entry here read
   // as "routed takes no arguments" and SILENTLY built a bare `routed()`, so a
   // production written to work both inside and outside a dispatch branch lost its

package/src/plugin/ir-eval.ts

@@ -2,6 +2,7 @@
  * The macro plugin's IR evaluator: turns a composed grammar's carried IR back
  * into combinators so the build can re-lower it. Build time only — see below.
  */
+import { columns, lines } from '../combinators/lines.ts'
 import type { Combinator } from '../types.ts'
 import { rules } from '../combinators/parser.ts'
 import { ref } from '../combinators/ref.ts'
@@ -155,11 +156,13 @@ function evalIR(ir: string, rulesImpl: unknown): unknown {
     'rules', 'ref', 'regex', 'literal', 'keywords', 'sequence', 'choice', 'dispatch', 'when', 'startsWith', 'endsWith', 'matches', 'otherwise', 'routed', 'attempt',
     'many', 'oneOrMore', 'optional', 'sepBy', 'keepSeparator', 'not', 'peek', 'node', 'parser',
     'scanTo', 'balanced', 'token', 'leaf', 'transform', 'trivia', 'classifiedTrivia', 'label', 'field', 'expect', 'adjacent', 'notAdjacent', '_tf', '_lf', '_nd', '_gch', '_wc',
+    'lines', 'columns',
     `return (${ir})`,
   )
   return fn(
     rulesImpl, ref, regex, literal, keywords, sequence, choice, dispatch, when, startsWith, endsWith, matches, otherwise, routed, attempt,
     many, oneOrMore, optional, sepBy, keepSeparator, not, peek, node, parser,
     scanTo, balanced, token, leaf, transform, trivia, classifiedTrivia, label, field, expectC, adjacent, notAdjacent, _tf, _lf, _nd, _gch, _wc,
+    lines, columns,
   )
 }

package/src/spec/model.ts

@@ -477,6 +477,21 @@ class Builder {
         return this.walk(def.parser)
       case 'scanTo':
         return { kind: 'annotation', text: '…' }
+      // A run of lines: each line opens with the prefix, the first with `first`
+      // when given, and the body reads the lines' contents.
+      case 'lines':
+        return {
+          kind: 'seq',
+          items: [
+            { kind: 'annotation', text: 'each line begins with' },
+            this.walk(def.prefix),
+            ...(def.first === undefined ? [] : [{ kind: 'annotation', text: 'the first with' } as SpecNode, this.walk(def.first)]),
+            ...(def.lazy === undefined ? [] : [{ kind: 'annotation', text: 'or continues where' } as SpecNode, this.walk(def.lazy)]),
+            this.walk(def.body),
+          ],
+        }
+      case 'columns':
+        return { kind: 'annotation', text: `${def.n} columns` }
 
       // Trivia — elided by default; included as an annotation when asked.
       case 'trivia':

package/src/table/assemble.ts

@@ -63,6 +63,8 @@
  * Semantics are `exec.ts`'s, case for case. That file remains the reference and
  * the three-way identity sweep gates this against it.
  */
+import { rebuildLineScan } from './line-scan.ts'
+import { scanRefs } from './program.ts'
 import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserDef } from '../types.ts'
 import { balanced, scanTo } from '../combinators/scanTo.ts'
 import { buildFieldMap } from '../compiler/fields.ts'
@@ -486,12 +488,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   }
 
   const labelExtraIps: number[] = []
-  for (const s of prog.scans ?? []) {
-    for (const r of s.skip) labelExtraIps.push(r[0])
-    if (s.sentinel !== undefined) labelExtraIps.push(s.sentinel[0])
-    if (s.recoverAt !== undefined) labelExtraIps.push(s.recoverAt[0])
-    if (s.stopAt !== undefined) labelExtraIps.push(s.stopAt[0])
-  }
+  for (const s of prog.scans ?? []) for (const r of scanRefs(s)) labelExtraIps.push(r[0])
   for (const set of prog.scanSkip ?? []) for (const r of set) labelExtraIps.push(r[0])
   const closureLabels = computeSiteLabels(
     code, [...Object.values(prog.rules), ...labelExtraIps], hostCst,
@@ -3781,12 +3778,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     // sites need emitted names too or `subtreeComb` would have to fall back to
     // a closure for them and the two engines would both be live in one parse.
     const extraIps: number[] = []
-    for (const s of prog.scans ?? []) {
-      for (const r of s.skip) extraIps.push(r[0])
-      if (s.sentinel !== undefined) extraIps.push(s.sentinel[0])
-      if (s.recoverAt !== undefined) extraIps.push(s.recoverAt[0])
-      if (s.stopAt !== undefined) extraIps.push(s.stopAt[0])
-    }
+    for (const s of prog.scans ?? []) for (const r of scanRefs(s)) extraIps.push(r[0])
     for (const set of prog.scanSkip ?? []) for (const r of set) extraIps.push(r[0])
     const roots = [...Object.values(prog.rules), ...extraIps]
     for (const ip of reachableSites(code, roots)) {
@@ -3964,6 +3956,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   }
 
   for (const s of prog.scans ?? []) {
+    const rebuilt = rebuildLineScan(s, subtreeComb)
+    if (rebuilt !== undefined) { scansArr.push(rebuilt); continue }
     const skip = s.skip.map(r => subtreeComb(r))
     const raw = (s.flags & 1) !== 0
     if (s.kind === 1) {

package/src/table/emit.ts

@@ -1,3 +1,4 @@
+import { scanRefs } from './program.ts'
 import type { FoldedProgram, TableProgram } from './program.ts'
 import { resolveTable } from './program.ts'
 import { EMITTED_PARAMS, Unemittable, emitAssemblySource } from './emit-assembly.ts'
@@ -109,6 +110,11 @@ function emitScanSpec(s: import('./program.ts').ScanSpec): string {
   if (s.sent !== undefined) parts.push(`sent:${s.sent === null ? 'null' : jsString(s.sent)}`)
   if (s.open !== undefined) parts.push(`open:${jsString(s.open)}`)
   if (s.close !== undefined) parts.push(`close:${jsString(s.close)}`)
+  for (const key of ['prefix', 'body', 'first', 'lazy'] as const) {
+    const r = s[key]
+    if (r !== undefined) parts.push(`${key}:${emitRef(r)}`)
+  }
+  if (s.n !== undefined) parts.push(`n:${s.n}`)
   return `{${parts.join(',')}}`
 }
 
@@ -196,12 +202,7 @@ function emitAssemblies(prog: TableProgram, cfgs: readonly RunCfg[]): string[] {
   // The scan pool and the scan-skip sets are linked from subtrees, so their
   // sites need emitted names too — same list `assemble.ts` builds.
   const extraIps: number[] = []
-  for (const s of prog.scans ?? []) {
-    for (const r of s.skip) extraIps.push(r[0])
-    if (s.sentinel !== undefined) extraIps.push(s.sentinel[0])
-    if (s.recoverAt !== undefined) extraIps.push(s.recoverAt[0])
-    if (s.stopAt !== undefined) extraIps.push(s.stopAt[0])
-  }
+  for (const s of prog.scans ?? []) for (const r of scanRefs(s)) extraIps.push(r[0])
   for (const set of prog.scanSkip ?? []) for (const r of set) extraIps.push(r[0])
 
   const out: string[] = []

package/src/table/encode.ts

@@ -1,3 +1,4 @@
+import { scanRefs } from './program.ts'
 import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts'
 import { classifyFinalChoice, firstSetOf, intersects, matchesEmpty, union, type RefResolver } from '../combinators/first-set.ts'
 import { childrenOf } from '../analysis/gating.ts'
@@ -683,6 +684,7 @@ class Encoder {
       case 'expect':
       case 'recover':
       case 'scanTo':
+      case 'lines':
       case 'unknown':
         return true
       case 'lazy': {
@@ -987,6 +989,17 @@ class Encoder {
       open: bal.open,
       close: bal.close,
     }))
+    // `lines()` and `columns()` lower as their constructor arguments too: the
+    // table rebuilds each and runs it, the body and prefixes being table subtrees.
+    if (d.tag === 'lines') {
+      return this.emit(OP_SCAN, this.scanSlot({
+        kind: 2, flags: 0, skip: [],
+        prefix: this.subtree(d.prefix), body: this.subtree(d.body),
+        ...(d.first === undefined ? {} : { first: this.subtree(d.first) }),
+        ...(d.lazy === undefined ? {} : { lazy: this.subtree(d.lazy) }),
+      }))
+    }
+    if (d.tag === 'columns') return this.emit(OP_SCAN, this.scanSlot({ kind: 3, flags: 0, skip: [], n: d.n }))
     if (d.tag === 'scanTo') {
       const sentDef = d.sentinel._def
       return this.emit(OP_SCAN, this.scanSlot({
@@ -1762,12 +1775,7 @@ class Encoder {
     // interiors uncollapsed and — worse — left the spec's own offset pointing at
     // a trampoline whose target had been rewritten around it.
     const refs: SubtreeRef[] = []
-    for (const s of this.scans) {
-      if (s.sentinel !== undefined) refs.push(s.sentinel)
-      if (s.recoverAt !== undefined) refs.push(s.recoverAt)
-      if (s.stopAt !== undefined) refs.push(s.stopAt)
-      refs.push(...s.skip)
-    }
+    for (const s of this.scans) refs.push(...scanRefs(s))
     for (const set of this.scanSkipSets) refs.push(...set)
     const stack = [...Object.values(this.rules), ...refs.map(r => r[0])]
     while (stack.length > 0) {
@@ -1788,6 +1796,10 @@ class Encoder {
       ...(s.sentinel === undefined ? {} : { sentinel: res(s.sentinel) }),
       ...(s.recoverAt === undefined ? {} : { recoverAt: res(s.recoverAt) }),
       ...(s.stopAt === undefined ? {} : { stopAt: res(s.stopAt) }),
+      ...(s.prefix === undefined ? {} : { prefix: res(s.prefix) }),
+      ...(s.body === undefined ? {} : { body: res(s.body) }),
+      ...(s.first === undefined ? {} : { first: res(s.first) }),
+      ...(s.lazy === undefined ? {} : { lazy: res(s.lazy) }),
     }))
     this.scanSkipSets = this.scanSkipSets.map(set => set.map(res))
   }

package/src/table/exec.ts

@@ -1,3 +1,4 @@
+import { rebuildLineScan } from './line-scan.ts'
 import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserDef } from '../types.ts'
 import { balanced, scanTo } from '../combinators/scanTo.ts'
 import { buildFieldMap } from '../compiler/fields.ts'
@@ -1682,6 +1683,8 @@ function makeDriver(
    * drift.
    */
   const scans: readonly Combinator<unknown>[] = (prog.scans ?? []).map(s => {
+    const rebuilt = rebuildLineScan(s, subtreeComb)
+    if (rebuilt !== undefined) return rebuilt
     const skip = s.skip.map(r => subtreeComb(r))
     const raw = (s.flags & 1) !== 0
     if (s.kind === 1) {

package/src/table/line-scan.ts

@@ -0,0 +1,19 @@
+import type { Combinator } from '../types.ts'
+import type { ScanSpec, SubtreeRef } from './program.ts'
+import { columns, lines } from '../combinators/lines.ts'
+
+/**
+ * Rebuild a `lines()` (kind 2) or `columns()` (kind 3) scan spec as the
+ * combinator it was constructed as, its subtrees bound to the table's pieces.
+ * Both drivers run the result through its own `parse`, as they do `scanTo`.
+ */
+export function rebuildLineScan(s: ScanSpec, subtree: (r: SubtreeRef) => Combinator<unknown>): Combinator<unknown> | undefined {
+  if (s.kind === 2) {
+    return lines(subtree(s.prefix!), subtree(s.body!), {
+      ...(s.first === undefined ? {} : { first: subtree(s.first) }),
+      ...(s.lazy === undefined ? {} : { lazy: subtree(s.lazy) }),
+    }) as Combinator<unknown>
+  }
+  if (s.kind === 3) return columns(s.n!) as Combinator<unknown>
+  return undefined
+}