parseman 0.46.0 → 0.47.1

package/src/cst/trivia-charscan.ts

@@ -0,0 +1,312 @@
+/**
+ * THE LABELLED-TRIVIA SCANNER, AT CHARACTER LEVEL.
+ *
+ * `fastTriviaScanner` (`../combinators/trivia-skip.ts`) already lowers UNLABELLED
+ * trivia to a char loop. Labelled trivia had no equivalent, so every gap between
+ * two sequence terms in a `classifiedTrivia()` grammar ran the arms as actual
+ * COMBINATORS — `matchArmAt` → `startsFirstSet` → a detached `ParseContext` → the
+ * regex combinator's `ParseResult`. On `benchmark.less`, one parse takes that path
+ * 31,758 times, tries 113,091 arms, and enters 13,696 of them, to classify 11,963
+ * chunks covering 33,981 characters — about 2.8 characters per chunk, for which the
+ * combinator machinery is pure overhead.
+ *
+ * The arms are static grammar data (`encode.ts`'s `triviaSpecOf` already lowers
+ * them to `[label, source, flags]`), so they can be classified ONCE, per spec, into
+ * per-arm character matchers. Anything that does not classify yields `null` and the
+ * combinator path stands — there is no fast path that is only usually right.
+ *
+ * A matcher returns the END of its match, or `pos` for "no match". That is exactly
+ * `matchArmAt`'s contract (`!r.ok || r.span.end <= pos` → null), so a lowered arm
+ * and a combinator arm are interchangeable inside the scan loops.
+ *
+ * The parser `state` `matchArmAt` threads into its detached context is not carried
+ * here, and does not need to be: only a plain `regex` arm lowers, and a regex
+ * terminal never reads state. An arm that could is exactly an arm that fails to
+ * classify.
+ */
+import type { Combinator } from '../types.ts'
+import { parseClassRanges } from '../regex/classes.ts'
+import type { LabeledTriviaSpec } from './trivia-kinds.ts'
+
+export type TriviaArmMatcher = (input: string, pos: number) => number
+
+type Ranges = Array<[number, number]>
+
+function inRanges(cp: number, ranges: Ranges): boolean {
+  for (let i = 0; i < ranges.length; i++) {
+    const r = ranges[i]!
+    if (cp >= r[0] && cp <= r[1]) return true
+  }
+  return false
+}
+
+/**
+ * A run of LITERAL characters as code points, or null.
+ *
+ * Only punctuation may be backslash-escaped: `\n`/`\d`/`\w` are not literals, and
+ * decoding them as one would silently accept a different language.
+ */
+function literalCodes(src: string): number[] | null {
+  const codes: number[] = []
+  for (let i = 0; i < src.length; i++) {
+    let c = src[i]!
+    if (c === '\\') {
+      c = src[i + 1] ?? ''
+      if (c === '' || /[A-Za-z0-9]/.test(c)) return null
+      i++
+    } else if (/[[\]()|*+?{}^$.]/.test(c)) {
+      return null
+    }
+    codes.push(c.charCodeAt(0))
+  }
+  return codes.length > 0 ? codes : null
+}
+
+/**
+ * ONE character as it appears inside a character class, where `*` and friends are
+ * ordinary characters. `\x`-style escapes still have to be punctuation.
+ */
+function classLiteralCode(src: string): number | null {
+  if (src.length === 1) return src.charCodeAt(0)
+  if (src.length === 2 && src[0] === '\\' && !/[A-Za-z0-9]/.test(src[1]!)) return src.charCodeAt(1)
+  return null
+}
+
+/** `[cls]+` / `[cls]*` — a positive character-class run. */
+function classifyClassRun(src: string): TriviaArmMatcher | null {
+  const m = /^\[([^\]^](?:[^\]]|\\.)*)\][*+]$/.exec(src)
+  if (!m) return null
+  const ranges = parseClassRanges(m[1]!)
+  if (!ranges) return null
+  // `*` can match empty, but an empty match is "no match" to every caller here
+  // (`matchArmAt` rejects `end <= pos`), so both quantifiers scan the same.
+  return (input, pos) => {
+    let p = pos
+    while (p < input.length && inRanges(input.charCodeAt(p), ranges)) p++
+    return p
+  }
+}
+
+/** `//[^\n\r]*` — a literal leader run to (not including) the line break. */
+function classifyLineRun(src: string): TriviaArmMatcher | null {
+  const m = /^(.*?)\[\^(?:\\n\\r|\\r\\n)\]\*$/.exec(src)
+  if (!m) return null
+  const codes = literalCodes(m[1]!)
+  if (!codes) return null
+  const n = codes.length
+  return (input, pos) => {
+    for (let i = 0; i < n; i++) if (input.charCodeAt(pos + i) !== codes[i]) return pos
+    let p = pos + n
+    while (p < input.length) {
+      const c = input.charCodeAt(p)
+      if (c === 10 || c === 13) break
+      p++
+    }
+    return p
+  }
+}
+
+/**
+ * `/*(?:[^*]|\*(?!\/))*\*\/` — a literal-delimited run, i.e. a block comment.
+ *
+ * The body alternation can never consume the closing pair, so the greedy regex
+ * stops at the FIRST close: an `indexOf` is not an approximation of it, it is the
+ * same position. An unterminated open fails in both, since the close is required.
+ */
+function classifyDelimited(src: string): TriviaArmMatcher | null {
+  const m = /^(\\?.)(\\?.)\(\?:\[\^(\\?.)\]\|(\\?.)\(\?!(\\?.)\)\)\*(\\?.)(\\?.)$/.exec(src)
+  if (!m) return null
+  const open = literalCodes(m[1]! + m[2]!)
+  const close = literalCodes(m[6]! + m[7]!)
+  // `m[3]` sits inside `[^…]`, where a regex metacharacter stands for itself and
+  // needs no backslash — so it is read with the class-body rule, not the strict one.
+  const bodyExcluded = classLiteralCode(m[3]!)
+  const bodyGuard = literalCodes(m[4]!)
+  const guardNext = literalCodes(m[5]!)
+  if (!open || !close || bodyExcluded === null || !bodyGuard || !guardNext) return null
+  // The body must exclude exactly the close pair and nothing else — the shape
+  // `(?:[^C0]|C0(?!C1))*` with `C0C1` the close.
+  if (bodyExcluded !== close[0] || bodyGuard[0] !== close[0] || guardNext[0] !== close[1]) return null
+  const closeStr = String.fromCharCode(...close)
+  const openLen = open.length
+  const closeLen = closeStr.length
+  return (input, pos) => {
+    for (let i = 0; i < openLen; i++) if (input.charCodeAt(pos + i) !== open[i]) return pos
+    const hit = input.indexOf(closeStr, pos + openLen)
+    return hit < 0 ? pos : hit + closeLen
+  }
+}
+
+/** Split a regex body on top-level `|`, tracking group depth and classes. */
+function splitAlts(body: string): string[] {
+  const out: string[] = []
+  let depth = 0
+  let inClass = false
+  let start = 0
+  for (let i = 0; i < body.length; i++) {
+    const c = body[i]
+    if (c === '\\') { i++; continue }
+    if (inClass) { if (c === ']') inClass = false; continue }
+    if (c === '[') inClass = true
+    else if (c === '(') depth++
+    else if (c === ')') depth--
+    else if (c === '|' && depth === 0) { out.push(body.slice(start, i)); start = i + 1 }
+  }
+  out.push(body.slice(start))
+  return out
+}
+
+/** Strip one full-span `(?:…)` wrapper. */
+function unwrapGroup(src: string): string {
+  if (!src.startsWith('(?:') || !src.endsWith(')')) return src
+  let depth = 0
+  for (let i = 0; i < src.length; i++) {
+    const c = src[i]
+    if (c === '\\') { i++; continue }
+    if (c === '(') depth++
+    else if (c === ')') { depth--; if (depth === 0) return i === src.length - 1 ? src.slice(3, -1) : src }
+  }
+  return src
+}
+
+/**
+ * `(?:A|B|C)+` — one arm that is itself a whole trivia alternation (the shape
+ * `classifiedTrivia({ whitespace: /…|…|…/ })` produces when a grammar gives every
+ * form the SAME label).
+ *
+ * ORDERED-GREEDY IS THE REGEX RESULT, not an approximation of it. A greedy `+`
+ * only reconsiders an alternative when the CONTINUATION fails, and here the
+ * continuation is "more repetitions, then end of pattern", which cannot fail once
+ * one repetition has matched. So the engine keeps the first alternative that
+ * matches at each position, exactly as this loop does — no disjointness needed
+ * (`//` and `/*` share a leading `/` and still agree). An alternative that matches
+ * empty terminates both, as `end > p` requires progress.
+ */
+function classifyAltRun(src: string): TriviaArmMatcher | null {
+  const m = /^\((?:\?:)?(.*)\)[*+]$/.exec(src)
+  if (!m) return null
+  const parts = splitAlts(m[1]!)
+  if (parts.length < 2) return null
+  const matchers: TriviaArmMatcher[] = []
+  for (const part of parts) {
+    const sub = classifyArm(unwrapGroup(part))
+    if (!sub) return null
+    matchers.push(sub)
+  }
+  const n = matchers.length
+  return (input, pos) => {
+    let p = pos
+    scan: while (p < input.length) {
+      for (let i = 0; i < n; i++) {
+        const end = matchers[i]!(input, p)
+        if (end > p) { p = end; continue scan }
+      }
+      break
+    }
+    return p
+  }
+}
+
+function classifyArm(src: string): TriviaArmMatcher | null {
+  return classifyClassRun(src)
+    ?? classifyLineRun(src)
+    ?? classifyDelimited(src)
+    ?? classifyAltRun(src)
+}
+
+function classifyParser(p: Combinator<unknown>): TriviaArmMatcher | null {
+  const d = p._def
+  if (d.tag !== 'regex' || d.flags) return null
+  return classifyArm(d.source)
+}
+
+let armSpec0: LabeledTriviaSpec | undefined
+let armSpec1: LabeledTriviaSpec | undefined
+let armSpec2: LabeledTriviaSpec | undefined
+let armSpec3: LabeledTriviaSpec | undefined
+let armValue0: readonly TriviaArmMatcher[] | null | undefined
+let armValue1: readonly TriviaArmMatcher[] | null | undefined
+let armValue2: readonly TriviaArmMatcher[] | null | undefined
+let armValue3: readonly TriviaArmMatcher[] | null | undefined
+
+/**
+ * The character-level matchers for a spec's arms — index-parallel to `spec.arms`,
+ * so a matcher's index IS its kind index — or `null` when any arm has no lowering.
+ *
+ * `spec.minRepeats > 1` is excluded: the loops below report only an END, and a run
+ * that fails a minimum of two must report NOTHING. `classifiedTrivia()` never
+ * produces one, and the combinator path still handles it if a host writes one.
+ */
+export function charArmsFor(spec: LabeledTriviaSpec): readonly TriviaArmMatcher[] | null {
+  if (spec === armSpec0) return armValue0!
+  if (spec === armSpec1) return armValue1!
+  if (spec === armSpec2) return armValue2!
+  if (spec === armSpec3) return armValue3!
+  let arms: TriviaArmMatcher[] | null = []
+  if (spec.minRepeats > 1) {
+    arms = null
+  } else {
+    for (const arm of spec.arms) {
+      const c = classifyParser(arm.parser)
+      if (!c) { arms = null; break }
+      arms.push(c)
+    }
+  }
+  armSpec3 = armSpec2
+  armValue3 = armValue2
+  armSpec2 = armSpec1
+  armValue2 = armValue1
+  armSpec1 = armSpec0
+  armValue1 = armValue0
+  armSpec0 = spec
+  armValue0 = arms
+  return arms
+}
+
+/** End of the trivia run at `cur`, or `cur` when nothing matched. */
+export function charTriviaEnd(input: string, cur: number, arms: readonly TriviaArmMatcher[]): number {
+  const n = arms.length
+  let pos = cur
+  scan: while (pos < input.length) {
+    for (let i = 0; i < n; i++) {
+      const end = arms[i]!(input, pos)
+      if (end > pos) { pos = end; continue scan }
+    }
+    break
+  }
+  return pos
+}
+
+/** As `charTriviaEnd`, reporting each chunk with the kind index that matched it. */
+export function charTriviaVisit(
+  input: string,
+  cur: number,
+  arms: readonly TriviaArmMatcher[],
+  visit: (start: number, end: number, kindIndex: number) => void,
+): number {
+  const n = arms.length
+  let pos = cur
+  scan: while (pos < input.length) {
+    for (let i = 0; i < n; i++) {
+      const end = arms[i]!(input, pos)
+      if (end > pos) { visit(pos, end, i); pos = end; continue scan }
+    }
+    break
+  }
+  return pos
+}
+
+/** The `1 << kindIndex` mask of the categories in the run at `cur`. */
+export function charTriviaKindMask(input: string, cur: number, arms: readonly TriviaArmMatcher[]): number {
+  const n = arms.length
+  let pos = cur
+  let mask = 0
+  scan: while (pos < input.length) {
+    for (let i = 0; i < n; i++) {
+      const end = arms[i]!(input, pos)
+      if (end > pos) { mask |= 1 << i; pos = end; continue scan }
+    }
+    break
+  }
+  return mask
+}

package/src/cst/trivia-entries.ts

@@ -0,0 +1,285 @@
+/** View over a flat trivia log (`_triviaLog` or per-node `triviaLog`). */
+export type TriviaEntriesView = {
+  readonly length: number
+  readonly labels: readonly string[] | undefined
+  readonly stride: number
+  start(i: number): number
+  end(i: number): number
+  /** Raw-child insertion boundary; defined only for a per-node trivia log. */
+  insertIndex(i: number): number | undefined
+  kindIndex(i: number): number | undefined
+  kind(i: number): string | undefined
+  text(i: number, input: string): string
+}
+
+export type RootTriviaGap = {
+  /** Start offset of the contiguous trivia gap. */
+  readonly start: number
+  /** End offset of the contiguous trivia gap. */
+  readonly end: number
+  /** Entry indices into `entries` that make up this gap. */
+  readonly entryIndices: readonly number[]
+  /** True when any entry in the gap has the requested labeled trivia kind. */
+  hasKind(kind: string): boolean
+  /** Slice the full contiguous gap text from the original input. */
+  text(input: string): string
+}
+
+export type RootTriviaIndex = {
+  /** Lazy entry-level view over sparse selected-root rows. */
+  readonly entries: TriviaEntriesView
+  /** Selected-root label table; entry kind indices refer to this array. */
+  readonly labels: readonly string[]
+  /**
+   * Contiguous trivia gaps keyed by the following content offset. Values are
+   * entry indices into `entries`, not materialized `{ value, span }` objects.
+   */
+  readonly before: ReadonlyMap<number, readonly number[]>
+  /**
+   * Contiguous trivia gaps keyed by the preceding content offset. Values are
+   * entry indices into `entries`, not materialized `{ value, span }` objects.
+   */
+  readonly after: ReadonlyMap<number, readonly number[]>
+  entryIndicesBefore(offset: number): readonly number[]
+  entryIndicesAfter(offset: number): readonly number[]
+  /**
+   * Contiguous trivia gap immediately before `offset`, keyed by the following
+   * content offset. Returns undefined when no gap sits at that boundary.
+   */
+  gapBefore(offset: number): RootTriviaGap | undefined
+  /**
+   * Contiguous trivia gap immediately after `offset`, keyed by the preceding
+   * content offset. Returns undefined when no gap sits at that boundary.
+   */
+  gapAfter(offset: number): RootTriviaGap | undefined
+  /** All contiguous trivia gaps in source order. */
+  gaps(): readonly RootTriviaGap[]
+  /** Source-ordered gaps containing at least one entry with the given label. */
+  gapsWithKind(kind: string | readonly string[]): readonly RootTriviaGap[]
+}
+
+const EMPTY_INDICES: readonly number[] = Object.freeze([])
+const EMPTY_GAPS: readonly RootTriviaGap[] = Object.freeze([])
+/** Number of packed fields in one selected-root trivia row. */
+export const SELECTED_ROOT_STRIDE = 5
+
+type RootTriviaMaps = {
+  before: Map<number, number[]>
+  after: Map<number, number[]>
+  beforeGaps: Map<number, RootTriviaGap>
+  afterGaps: Map<number, RootTriviaGap>
+  gaps: RootTriviaGap[]
+}
+
+function entryOffset(i: number, stride: number): number {
+  return i * stride
+}
+
+/**
+ * Wrap a flat trivia log. Stride is inferred from `labels`:
+ * - root `_triviaLog`: 2 (start, end) or 3 (+ kindIndex)
+ * - node `triviaLog`: 3 (start, end, insertIdx) or 4 (+ kindIndex)
+ */
+export function triviaEntries(
+  log: readonly number[],
+  labels?: readonly string[],
+  opts?: { nodeLog?: boolean },
+): TriviaEntriesView {
+  const baseStride = opts?.nodeLog ? 3 : 2
+  const stride = labels ? baseStride + 1 : baseStride
+  const length = Math.floor(log.length / stride)
+
+  return {
+    length,
+    labels,
+    stride,
+    start(i) {
+      return log[entryOffset(i, stride)]!
+    },
+    end(i) {
+      return log[entryOffset(i, stride) + 1]!
+    },
+    insertIndex(i) {
+      return opts?.nodeLog ? log[entryOffset(i, stride) + 2] : undefined
+    },
+    kindIndex(i) {
+      if (!labels) return undefined
+      return log[entryOffset(i, stride) + baseStride]
+    },
+    kind(i) {
+      const ki = labels ? log[entryOffset(i, stride) + baseStride] : undefined
+      return ki !== undefined ? labels![ki] : undefined
+    },
+    text(i, input) {
+      const s = log[entryOffset(i, stride)]!
+      const e = log[entryOffset(i, stride) + 1]!
+      return input.slice(s, e)
+    },
+  }
+}
+
+/**
+ * Build a root view over the sparse selected-label rows captured as
+ * `[ownedRangeStart, ownedRangeEnd, markerStart, markerEnd, kindIndex]`.
+ *
+ * This is intentionally a different input shape from `triviaLog`: each row is
+ * one selected marker, while the first pair names the complete committed trivia
+ * range around it. Thus a mixed trivia run costs one five-number row, not three
+ * legacy trivia entries. Rows are source ordered and equal gap pairs are
+ * contiguous, which lets singleton boundary lookups binary-search the sparse
+ * rows without constructing document-wide maps.
+ */
+export function buildRootTriviaIndex(
+  log: readonly number[],
+  labels: readonly string[],
+): RootTriviaIndex {
+  const length = Math.floor(log.length / SELECTED_ROOT_STRIDE)
+  const markerOffset = (i: number) => i * SELECTED_ROOT_STRIDE
+  const ownedRangeStart = (i: number) => log[markerOffset(i)]!
+  const ownedRangeEnd = (i: number) => log[markerOffset(i) + 1]!
+  const markerStart = (i: number) => log[markerOffset(i) + 2]!
+  const markerEnd = (i: number) => log[markerOffset(i) + 3]!
+  const markerKind = (i: number) => log[markerOffset(i) + 4]!
+
+  const entries: TriviaEntriesView = {
+    length,
+    labels,
+    stride: SELECTED_ROOT_STRIDE,
+    start: markerStart,
+    end: markerEnd,
+    insertIndex: () => undefined,
+    kindIndex: markerKind,
+    kind(i) {
+      return labels[markerKind(i)]
+    },
+    text(i, input) {
+      return input.slice(markerStart(i), markerEnd(i))
+    },
+  }
+
+  const lowerBound = (offset: number, by: (i: number) => number): number => {
+    let lo = 0
+    let hi = length
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1
+      if (by(mid) < offset) lo = mid + 1
+      else hi = mid
+    }
+    return lo
+  }
+
+  const gapRowsAt = (offset: number, direction: 'before' | 'after'): [number, number] | undefined => {
+    const edge = direction === 'before' ? ownedRangeEnd : ownedRangeStart
+    let first = lowerBound(offset, edge)
+    if (first === length || edge(first) !== offset) return undefined
+    const start = ownedRangeStart(first)
+    const end = ownedRangeEnd(first)
+    while (first > 0 && ownedRangeStart(first - 1) === start && ownedRangeEnd(first - 1) === end) first--
+    let last = first + 1
+    while (last < length && ownedRangeStart(last) === start && ownedRangeEnd(last) === end) last++
+    return [first, last]
+  }
+
+  const indices = (first: number, last: number): readonly number[] => {
+    const out: number[] = []
+    for (let i = first; i < last; i++) out.push(i)
+    return out
+  }
+
+  const makeGap = (first: number, last: number): RootTriviaGap => {
+    const start = ownedRangeStart(first)
+    const end = ownedRangeEnd(first)
+    let entryIndices: readonly number[] | undefined
+    return {
+      start,
+      end,
+      get entryIndices() {
+        return entryIndices ??= indices(first, last)
+      },
+      hasKind(kind) {
+        for (let i = first; i < last; i++) {
+          if (labels[markerKind(i)] === kind) return true
+        }
+        return false
+      },
+      text(input) {
+        return input.slice(start, end)
+      },
+    }
+  }
+
+  let maps: RootTriviaMaps | undefined
+  const getMaps = () => {
+    if (maps !== undefined) return maps
+    const before = new Map<number, number[]>()
+    const after = new Map<number, number[]>()
+    const beforeGaps = new Map<number, RootTriviaGap>()
+    const afterGaps = new Map<number, RootTriviaGap>()
+    const gaps: RootTriviaGap[] = []
+    for (let first = 0; first < length;) {
+      let last = first + 1
+      while (last < length && ownedRangeStart(last) === ownedRangeStart(first) && ownedRangeEnd(last) === ownedRangeEnd(first)) last++
+      const gap = makeGap(first, last)
+      gaps.push(gap)
+      afterGaps.set(gap.start, gap)
+      beforeGaps.set(gap.end, gap)
+      after.set(gap.start, [...gap.entryIndices])
+      before.set(gap.end, [...gap.entryIndices])
+      first = last
+    }
+    maps = { before, after, beforeGaps, afterGaps, gaps }
+    return maps
+  }
+
+  return {
+    entries,
+    labels,
+    get before() { return getMaps().before },
+    get after() { return getMaps().after },
+    entryIndicesBefore(offset) {
+      const rows = gapRowsAt(offset, 'before')
+      return rows ? indices(rows[0], rows[1]) : EMPTY_INDICES
+    },
+    entryIndicesAfter(offset) {
+      const rows = gapRowsAt(offset, 'after')
+      return rows ? indices(rows[0], rows[1]) : EMPTY_INDICES
+    },
+    gapBefore(offset) {
+      const rows = gapRowsAt(offset, 'before')
+      return rows ? makeGap(rows[0], rows[1]) : undefined
+    },
+    gapAfter(offset) {
+      const rows = gapRowsAt(offset, 'after')
+      return rows ? makeGap(rows[0], rows[1]) : undefined
+    },
+    gaps() {
+      return getMaps().gaps
+    },
+    gapsWithKind(kind) {
+      const kinds = typeof kind === 'string' ? [kind] : kind
+      const matches: RootTriviaGap[] = []
+      // Do not materialize the document-wide boundary maps for a sparse query.
+      // The primary consumer selects comments: it needs only comment-bearing gaps,
+      // not an object/Map for every whitespace-owned boundary in the document.
+      for (let first = 0; first < length;) {
+        let last = first + 1
+        while (last < length
+          && ownedRangeStart(last) === ownedRangeStart(first)
+          && ownedRangeEnd(last) === ownedRangeEnd(first)) last++
+        let matched = false
+        for (let i = first; i < last && !matched; i++) {
+          const label = labels[markerKind(i)]
+          for (const kind of kinds) {
+            if (label === kind) {
+              matched = true
+              break
+            }
+          }
+        }
+        if (matched) matches.push(makeGap(first, last))
+        first = last
+      }
+      return matches.length === 0 ? EMPTY_GAPS : matches
+    },
+  }
+}