parseman 0.57.0 → 0.58.1

package/src/table/factor-choice.ts

@@ -0,0 +1,173 @@
+/**
+ * SHARED ENDS, FACTORED — a choice whose arms differ in one term.
+ *
+ * `choice(leaf(noTrivia(sequence(gap, '*', gap))), leaf(noTrivia(sequence(gap, '+', gap))))`
+ * scans `gap` once per arm, and on a value with no operator (`red red`) both
+ * arms scan it and then fail. Every arm is the same wrappers over a sequence of
+ * the same terms except one, so the choice is that one term:
+ *
+ *   leaf(noTrivia(sequence(gap, choice('*', '+'), gap)))
+ *
+ * The gap is scanned once, and the inner choice dispatches on the character
+ * after it.
+ *
+ * WHY IT IS THE SAME PARSE. Each arm is tried at the same position with the same
+ * context, so its terms before the differing one match the same text with the
+ * same value in every arm: running them once is what each arm's retry did. The
+ * inner choice tries the differing terms in arm order, as the outer one tried
+ * the arms. The terms after it may differ only as `optional`/`many`, which
+ * cannot fail, so the arm that matched its differing term is the arm the outer
+ * choice took. The sequence's values, the wrappers' spans and the leaf or node a
+ * wrapper publishes are what that arm produced.
+ *
+ * A choice that fails rolls its last arm back and reports the position it started
+ * at, so the factored form is an `attempt`, which does both. An arm that may
+ * commit is left as written: a committed choice reports without rolling back.
+ *
+ * WHAT "THE SAME" MEANS. Two terms are the same when they are one combinator, or
+ * their definitions are, field for field, down to shared combinators. A reducer
+ * is the same function, or the same printed source: a table artifact carries a
+ * reducer AS its source, so two reducers printed alike are one function there.
+ */
+import type { Combinator, ParserDef } from '../types.ts'
+import { choice } from '../combinators/choice.ts'
+import { sequence } from '../combinators/sequence.ts'
+import { leaf, sourceLeaf } from '../combinators/token.ts'
+import { noTrivia, parser } from '../combinators/grammar.ts'
+import { attempt } from '../combinators/attempt.ts'
+import { mayCommitFailure } from '../analysis/commitment.ts'
+
+type ChoiceDef = Extract<ParserDef, { tag: 'choice' }>
+
+/** Own keys every combinator may carry beside its definition; all derived from it. */
+const DERIVED = new Set(['_tag', '_meta', '_def', '_parseScalar', '_stickyRegex', 'parse'])
+
+function isCombinator(v: unknown): v is Combinator<unknown> {
+  return typeof v === 'object' && v !== null && '_def' in v && '_meta' in v
+}
+
+function sameComb(a: Combinator<unknown>, b: Combinator<unknown>, depth: number): boolean {
+  if (a === b) return true
+  if (depth > 32 || a._tag !== b._tag) return false
+  for (const k of Object.keys(a)) if (!DERIVED.has(k)) return false
+  for (const k of Object.keys(b)) if (!DERIVED.has(k)) return false
+  return sameFields(a._def as Record<string, unknown>, b._def as Record<string, unknown>, depth, undefined)
+}
+
+function sameValue(u: unknown, v: unknown, depth: number): boolean {
+  if (u === v) return true
+  if (isCombinator(u) && isCombinator(v)) return sameComb(u, v, depth + 1)
+  if (Array.isArray(u) && Array.isArray(v)) return u.length === v.length && u.every((x, i) => sameValue(x, v[i], depth))
+  if (typeof u === 'object' && u !== null && typeof v === 'object' && v !== null
+    && Object.getPrototypeOf(u) === Object.prototype && Object.getPrototypeOf(v) === Object.prototype) {
+    return sameFields(u as Record<string, unknown>, v as Record<string, unknown>, depth, undefined)
+  }
+  return false
+}
+
+/** Every field of `x` and `y` the same, `skip` aside. A reducer matches its twin by printed source. */
+function sameFields(x: Record<string, unknown>, y: Record<string, unknown>, depth: number, skip: string | undefined): boolean {
+  const kx = Object.keys(x).filter(k => k !== skip && x[k] !== undefined)
+  const ky = Object.keys(y).filter(k => k !== skip && y[k] !== undefined)
+  if (kx.length !== ky.length) return false
+  for (const k of kx) {
+    if (!(k in y)) return false
+    const u = x[k]
+    const v = y[k]
+    if (k === 'fn' && typeof u === 'function' && typeof v === 'function' && u !== v) {
+      if (typeof x.fnSrc !== 'string' || x.fnSrc === '' || x.fnSrc !== y.fnSrc) return false
+      continue
+    }
+    if (!sameValue(u, v, depth)) return false
+  }
+  return true
+}
+
+/** A wrapper the factoring can rebuild around a new body: a leaf, or a trivia scope. */
+function rewrap(w: Combinator<unknown>, body: Combinator<unknown>): Combinator<unknown> | undefined {
+  const d = w._def
+  let out: Combinator<unknown>
+  if (d.tag === 'leaf') {
+    out = d.cstValue === 'source' ? sourceLeaf(body, d.fn) : leaf(body, d.fn)
+    if (d.fnSrc !== undefined) (out._def as { fnSrc?: string }).fnSrc = d.fnSrc
+  } else if (d.tag === 'grammar') {
+    out = d.clearTrivia === true ? noTrivia(body) : parser({ trivia: d.triviaParser ?? null }, body)
+  } else return undefined
+  // The rebuilt wrapper must be the original in everything but its body.
+  return sameFields(d as Record<string, unknown>, out._def as Record<string, unknown>, 0, 'parser') ? out : undefined
+}
+
+function unusedOf(seq: Combinator<unknown>): boolean {
+  return seq._def.tag === 'sequence' && seq._def.valueUnused === true
+}
+
+function never(c: Combinator<unknown>): boolean {
+  const t = c._def.tag
+  return t === 'optional' || t === 'many'
+}
+
+/**
+ * The factored form of `d`'s choice, or `undefined` when its arms are not one
+ * shape that differs in one term.
+ */
+export function factorSharedEnds(d: ChoiceDef): Combinator<unknown> | undefined {
+  const arms = d.parsers
+  if (arms.length < 2) return undefined
+  if (d.strategy.tag !== 'firstMatch' && d.strategy.tag !== 'sharedPrefix') return undefined
+  if (d.gates.some(g => g !== null) || d.autoNot.some(a => a !== null)) return undefined
+  // Each arm: its wrappers, outermost first, and the sequence they end in.
+  const shapes: { wrappers: Combinator<unknown>[]; terms: Combinator<unknown>[]; seq: Combinator<unknown> }[] = []
+  for (const arm of arms) {
+    const wrappers: Combinator<unknown>[] = []
+    let c = arm
+    while ((c._def.tag === 'leaf' || c._def.tag === 'grammar') && wrappers.length < 4) {
+      wrappers.push(c)
+      c = c._def.parser
+    }
+    if (c._def.tag !== 'sequence' || c._def.parsers.length < 2) return undefined
+    shapes.push({ wrappers, terms: c._def.parsers, seq: c })
+  }
+  const first = shapes[0]!
+  const n = first.terms.length
+  for (const s of shapes) {
+    if (s.wrappers.length !== first.wrappers.length || s.terms.length !== n) return undefined
+    if (unusedOf(s.seq) !== unusedOf(first.seq)) return undefined
+    for (let j = 0; j < s.wrappers.length; j++) {
+      const a = first.wrappers[j]!._def as Record<string, unknown>
+      const b = s.wrappers[j]!._def as Record<string, unknown>
+      if (a.tag !== b.tag || !sameFields(a, b, 0, 'parser')) return undefined
+    }
+  }
+  // Exactly one term position differs, and nothing after it can fail.
+  let at = -1
+  for (let i = 0; i < n; i++) {
+    if (shapes.every(s => sameComb(s.terms[i]!, first.terms[i]!, 0))) continue
+    if (at >= 0) return undefined
+    at = i
+  }
+  if (at < 0) return undefined
+  for (let i = at + 1; i < n; i++) if (!never(first.terms[i]!)) return undefined
+  // A committed failure leaves a choice without rolling its arm back. Asked last:
+  // it walks everything the arms reach.
+  if (arms.some(a => mayCommitFailure(a))) return undefined
+  const middle = choice(...(shapes.map(s => s.terms[at]!) as [Combinator<unknown>, ...Combinator<unknown>[]]))
+  // The inner choice must keep arm order: a strategy that reorders (longest
+  // literal first) or classifies would pick a different arm. Nor may it add an
+  // automatic lookahead the arms as written do not have: `'start'` before
+  // `'started'` gets a `not('ed')` as one choice, which rejects the `!start` the
+  // written arms take on `!started`.
+  if (middle._def.tag === 'choice' && middle._def.autoNot.some(a => a !== null)) return undefined
+  const st = middle._def.tag === 'choice' ? middle._def.strategy : undefined
+  if (st === undefined || (st.tag !== 'firstMatch' && st.tag !== 'sharedPrefix'
+    && !(st.tag === 'literalsLongestFirst' && st.sortedIndices.every((x, i) => x === i)))) return undefined
+  let out: Combinator<unknown> | undefined = sequence(
+    ...([...first.terms.slice(0, at), middle, ...first.terms.slice(at + 1)] as [Combinator<unknown>, ...Combinator<unknown>[]]),
+  )
+  if (unusedOf(first.seq)) {
+    (out._def as { valueUnused?: boolean }).valueUnused = true
+  }
+  for (let j = first.wrappers.length - 1; j >= 0 && out !== undefined; j--) out = rewrap(first.wrappers[j]!, out)
+  // A choice that fails is rolled back and reports where it started; so does an
+  // `attempt`. The sequence alone would report where its inner choice failed.
+  return out === undefined ? undefined : attempt(out)
+}

package/src/table/fail-clean.ts

@@ -0,0 +1,130 @@
+/**
+ * WHICH ROWS FAIL CLEAN.
+ *
+ * A choice, an `optional()` and a repeat item take a rollback mark before they
+ * try their child and restore it when the child fails, so a failed attempt leaves
+ * nothing in the capture sinks: the open frame's children, raw entries and trivia
+ * rows, the fields, the errors and the two trivia logs. Most children never write
+ * to a sink before they fail. A terminal fails before it pushes its leaf; a node
+ * that fails drops its own frame; a sequence whose later terms cannot fail can
+ * only fail in its first term, which has no trivia in front of it. For such a
+ * child the mark and the restore are work with nothing to undo.
+ *
+ * `failsClean(ip)` answers, from the table alone, whether every failure of the
+ * row at `ip` leaves every sink as the row found it. Only uncommitted failures
+ * count: a committed failure is propagated past every choice, optional and
+ * repeat without their restore, so the mark is never read for it.
+ *
+ * It is one memoised walk, run while the assembler links, so it is paid on the
+ * first parse. A rule reached again while it is still being answered counts as
+ * dirty, and so does a row whose behaviour is not modelled here: `false` keeps
+ * the mark, so every approximation errs towards the old cost, never a wrong tree.
+ */
+import {
+  OP_ATTEMPT, OP_CHOICE, OP_COV, OP_DISPATCH, OP_EMPTY, OP_EXPECT, OP_FIELD, OP_GATE, OP_GUARD, OP_LABEL,
+  OP_LEAF, OP_LIT, OP_LIT_CI, OP_LIT_CI_TRACK, OP_LIT_TRACK, OP_NODE, OP_NODE_TRACK, OP_NOT, OP_OPT, OP_PEEK,
+  OP_REP, OP_REPV, OP_ROUTED, OP_RULE, OP_RX, OP_RX_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ,
+  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM,
+} from './ops.ts'
+
+export type FailClean = (ip: number) => boolean
+
+/**
+ * The child offset of a row that fails exactly when its one child fails and
+ * writes nothing of its own first, or −1.
+ */
+function passChild(code: ArrayLike<number>, ip: number): number {
+  switch (code[ip]) {
+    case OP_RULE: case OP_LABEL: case OP_COV: case OP_TOKEN:
+      return code[ip + 1]!
+    case OP_GATE: case OP_NODE: case OP_NODE_TRACK: case OP_XFORM: case OP_FIELD: case OP_SCOPE:
+    case OP_SCOPE_CAP: case OP_SCOPE_PLAIN: case OP_WITHCTX: case OP_LEAF: case OP_SOURCE_LEAF:
+      return code[ip + 2]!
+    default:
+      return -1
+  }
+}
+
+/** Sequence terms: `[first offset, count]` packed as the offset of the count word, or −1. */
+function seqCount(code: ArrayLike<number>, ip: number): number {
+  const op = code[ip]
+  return op === OP_SEQ || op === OP_SEQV ? ip + 1 : op === OP_SEQX ? ip + 2 : -1
+}
+
+export function failCleanAnalysis(code: ArrayLike<number>): FailClean {
+  // Per row: 0 unknown, 1 yes, 2 no, 3 being answered (answered as no).
+  const cleanAt = new Uint8Array(code.length)
+  const neverAt = new Uint8Array(code.length)
+
+  /** No uncommitted failure at all. */
+  const neverFails = (ip: number): boolean => {
+    const hit = neverAt[ip]!
+    if (hit !== 0) return hit === 1
+    neverAt[ip] = 3
+    let out = false
+    const op = code[ip]
+    if (op === OP_OPT || op === OP_EMPTY || op === OP_EXPECT) out = true
+    else if (op === OP_REP || op === OP_REPV) out = code[ip + 2] === 0
+    else {
+      const at = seqCount(code, ip)
+      if (at >= 0) {
+        out = true
+        for (let i = 1, n = code[at]!; out && i <= n; i++) out = neverFails(code[at + i]!)
+      } else if (op !== OP_GATE) {
+        const c = passChild(code, ip)
+        out = c >= 0 && neverFails(c)
+      }
+    }
+    neverAt[ip] = out ? 1 : 2
+    return out
+  }
+
+  const clean = (ip: number): boolean => {
+    const hit = cleanAt[ip]!
+    if (hit !== 0) return hit === 1
+    cleanAt[ip] = 3
+    let out = false
+    switch (code[ip]) {
+      // A terminal pushes its leaf only on a match; NOT/PEEK/ATTEMPT restore
+      // their own mark; OPT, EMPTY and EXPECT do not fail uncommitted.
+      case OP_LIT: case OP_RX: case OP_LIT_TRACK: case OP_RX_TRACK: case OP_LIT_CI: case OP_LIT_CI_TRACK:
+      case OP_GUARD: case OP_NOT: case OP_PEEK: case OP_ATTEMPT: case OP_EMPTY: case OP_OPT: case OP_EXPECT:
+        out = true
+        break
+      // Min 0 does not fail uncommitted; min 1 fails only with no item kept, and
+      // restores the failed item itself.
+      case OP_REP: case OP_REPV:
+        out = code[ip + 2]! <= 1
+        break
+      // Only the first term may fail: a later one fails after the terms before it
+      // wrote, and after the trivia scanned in front of it.
+      case OP_SEQ: case OP_SEQV: case OP_SEQX: {
+        const at = seqCount(code, ip)
+        const n = code[at]!
+        out = n > 0 && clean(code[at + 1]!)
+        for (let i = 2; out && i <= n; i++) out = neverFails(code[at + i]!)
+        break
+      }
+      case OP_CHOICE: {
+        out = true
+        for (let i = 0, n = code[ip + 2]!; out && i < n; i++) out = clean(code[ip + 4 + i]!)
+        break
+      }
+      case OP_ROUTED:
+        out = code[ip + 1]! < 0 || clean(code[ip + 1]!)
+        break
+      // An arm's failure and the fallback's are committed; only the selector's is
+      // not, and without a fallback an unknown key fails after the selector wrote.
+      case OP_DISPATCH:
+        out = code[ip + 3]! >= 0 && clean(code[ip + 1]!)
+        break
+      default: {
+        const c = passChild(code, ip)
+        out = c >= 0 && clean(c)
+      }
+    }
+    cleanAt[ip] = out ? 1 : 2
+    return out
+  }
+  return clean
+}

package/src/table/hold-gap.ts

@@ -0,0 +1,68 @@
+/**
+ * A GAP HELD OVER A LOOKAHEAD.
+ *
+ * In a sequence, the trivia before each term is scanned and recorded, and when
+ * the term matches nothing it is unrecorded and the cursor goes back before it:
+ * the next term scans the same gap again. Where the term that matched nothing is
+ * a lookahead (`not`/`peek`, always zero-width) and the next term cannot match
+ * empty, that second scan reads the same characters from the same position under
+ * the same trivia and records the same rows, and the next term then either
+ * consumes past the gap or the sequence fails. So the gap is kept: the lookahead
+ * runs at the gap's end, the cursor stays there, and the next term starts there
+ * with nothing left to scan. One scan where there were two and an undo.
+ *
+ * Not before an adjacency assertion or a leading-trivia field, which read the
+ * cursor from before the gap.
+ */
+import {
+  OP_ADJ, OP_ATTEMPT, OP_CHOICE, OP_COV, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_LIT_CI,
+  OP_NODE, OP_NOT, OP_PEEK, OP_REP, OP_REPV, OP_RULE, OP_RX, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ,
+  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM,
+} from './ops.ts'
+import { childSlots } from './child-slots.ts'
+import { leadingTriviaFieldAt } from './field-spec.ts'
+import { regexCanMatchEmpty } from '../regex/first-set.ts'
+
+const WRAPPERS = new Set([OP_NODE, OP_XFORM, OP_LEAF, OP_SOURCE_LEAF, OP_TOKEN, OP_SCOPE, OP_SCOPE_CAP,
+  OP_SCOPE_PLAIN, OP_FIELD, OP_LABEL, OP_EXPECT, OP_WITHCTX, OP_RULE, OP_ATTEMPT, OP_COV])
+
+/**
+ * Can the row at `ip` match empty? `true` wherever it cannot tell. A first-set
+ * gate row wraps only a child that cannot (`encode.ts` gates nothing nullable).
+ */
+export function rowNullable(code: ArrayLike<number>, k: readonly unknown[], ip: number, memo: Map<number, boolean>): boolean {
+  const hit = memo.get(ip)
+  if (hit !== undefined) return hit
+  memo.set(ip, true) // a cycle is answered conservatively
+  const op = code[ip]!
+  const kids: number[] = []
+  let out = true
+  if (op === OP_GATE) out = false
+  else if (op === OP_LIT || op === OP_LIT_CI) out = (k[code[ip + 1]!] as string).length === 0
+  else if (op === OP_RX) { const re = k[code[ip + 1]!]; out = !(re instanceof RegExp) || regexCanMatchEmpty(re.source) }
+  else if (op === OP_SEQ || op === OP_SEQV || op === OP_SEQX) {
+    childSlots(code, ip, kids)
+    out = kids.every(c => rowNullable(code, k, c, memo))
+  } else if (op === OP_CHOICE) {
+    childSlots(code, ip, kids)
+    out = kids.length === 0 || kids.some(c => rowNullable(code, k, c, memo))
+  } else if (op === OP_REP || op === OP_REPV) {
+    out = code[ip + 2] === 0 || rowNullable(code, k, code[ip + 1]!, memo)
+  } else if (WRAPPERS.has(op)) {
+    if (childSlots(code, ip, kids) && kids.length === 1) out = rowNullable(code, k, kids[0]!, memo)
+  }
+  memo.set(ip, out)
+  return out
+}
+
+/** Term `i` of the sequence at `seq` (terms from `base`, `n` of them) holds its gap. */
+export function holdsGap(
+  code: ArrayLike<number>, k: readonly unknown[], base: number, n: number, i: number, memo: Map<number, boolean>,
+): boolean {
+  if (i < 1 || i + 1 >= n) return false
+  const term = code[base + i]!
+  if (code[term] !== OP_NOT && code[term] !== OP_PEEK) return false
+  const next = code[base + i + 1]!
+  if (code[next] === OP_ADJ || leadingTriviaFieldAt(code, k, next)) return false
+  return !rowNullable(code, k, next, memo)
+}