parseman 0.59.0 → 0.59.1

package/src/compiler/token-scanner.ts

@@ -35,7 +35,7 @@
 import type { Alphabet } from './token-alphabet.ts'
 import { TOK_EOF, TOK_UNKNOWN, TOK_WS } from './token-alphabet.ts'
 import { firstSetFromRegex } from '../regex/first-set.ts'
-import { packInts, sharedHelperDecl } from './token-dispatch.ts'
+import { packInts, unpackHelperDecl } from './token-dispatch.ts'
 import { foldCode, foldExpr } from './token-dispatch.ts'
 
 /**
@@ -193,7 +193,7 @@ export function emitScanner(alphabet: Alphabet, ns: string): ScannerEmission {
   // two halves of one encoding still lived in two files. They agree today — the
   // point is that widening the encoding (which the `RangeError` tells you to do)
   // must not be able to change one half and miss the other.
-  decls.push(sharedHelperDecl('unpack', () => p('_tkUnpack')))
+  decls.push(unpackHelperDecl(p('_tkUnpack')))
   decls.push(`const ${p('_tkNext_')} = ${p('_tkUnpack')}(${packInts(NEXT)})`)
   decls.push(`const ${p('_tkStart')} = ${p('_tkUnpack')}(${packInts(START)})`)
   decls.push(`const ${p('_tkAccept')} = ${p('_tkUnpack')}(${packInts(ACCEPT)})`)

package/src/cst/root-trivia-scope.ts

@@ -14,7 +14,7 @@
 
 /** The message `parser()` raises for an unclassified local scope. Exported so the
  *  three engines cannot drift a word apart. */
-export const ROOT_TRIVIA_SCOPE_REFUSAL
+const ROOT_TRIVIA_SCOPE_REFUSAL
   = 'parser(): selected root trivia requires classifiedTrivia() for every local trivia scope, or rootCapture: \'opaque\'.'
 
 /**

package/src/cst/trivia-entries.ts

@@ -61,7 +61,7 @@ export type RootTriviaIndex = {
 const EMPTY_INDICES: readonly number[] = Object.freeze([])
 const EMPTY_GAPS: readonly RootTriviaGap[] = Object.freeze([])
 /** Number of packed fields in one selected-root trivia row. */
-export const SELECTED_ROOT_STRIDE = 5
+const SELECTED_ROOT_STRIDE = 5
 
 type RootTriviaMaps = {
   before: Map<number, number[]>

package/src/cst/trivia-kinds.ts

@@ -60,7 +60,7 @@ function unwrapTrivia(p: Combinator<unknown>): Combinator<unknown> {
 }
 
 /** Strip `label()` wrapper; returns inner parser for matching. */
-export function peelLabel(p: Combinator<unknown>): { label: string; parser: Combinator<unknown> } | null {
+function peelLabel(p: Combinator<unknown>): { label: string; parser: Combinator<unknown> } | null {
   if (p._def.tag === 'label') return { label: p._def.label, parser: p._def.parser }
   return null
 }

package/src/spec/model.ts

@@ -247,7 +247,7 @@ function fixedString(alt: string): string | undefined {
  * boundary (`word()` / `keywords()` and every hand-written copy of them), and a
  * `\b…\b`-anchored word.
  */
-export function fixedStringsOfRegex(source: string): string[] | undefined {
+function fixedStringsOfRegex(source: string): string[] | undefined {
   let src = source
   // `\b` anchors assert a boundary, exactly like the `(?![…])` guard.
   if (src.startsWith('\\b')) src = src.slice(2)

package/src/table/encode.ts

@@ -517,18 +517,10 @@ class Encoder {
   /** TOKEN bodies are an all-program decision. A complete occurrence inside an
    * otherwise incomplete final graph stays on CHARACTER with every sibling. */
   readonly selectLexicalBodies: boolean
-  /**
-   * Decides the extra operands laid down below. ALWAYS TRUE — see
-   * `TableSettings.recovery`. Kept as a field rather than folded away because it
-   * is what every emit site reads, and the emit sites are where the shape of the
-   * row is stated.
-   */
-  readonly rec: boolean
   constructor(settings: TableSettings, selectLexicalBodies: boolean) {
     this.settings = settings
     this.track = settings.trackLines === true
     this.selectLexicalBodies = selectLexicalBodies
-    this.rec = true
     this.plan = settings.coverage
     this.cov = settings.coverage === undefined
       ? undefined
@@ -1147,16 +1139,12 @@ class Encoder {
       }
       case 'sequence': {
         const kids = d.parsers.map(c => this.node(c).ip)
-        // A RECOVERY table appends one follow-set class per term AFTER the child
-        // slots. Appending keeps every existing read (`ip + 2 + i`) and the
-        // `collapseIndirection` slot map exactly as they were, so a strict table
-        // is word-for-word the table it always was.
-        const head = this.emitHead(d.valueUnused ? OP_SEQV : OP_SEQ, 1 + kids.length + (this.rec ? kids.length : 0))
+        // One follow-set class per term (recovery's resync sentinel) follows the
+        // child slots, so every child read stays `ip + 2 + i`.
+        const head = this.emitHead(d.valueUnused ? OP_SEQV : OP_SEQ, 1 + 2 * kids.length)
         this.code[head + 1] = kids.length
         for (let i = 0; i < kids.length; i++) this.code[head + 2 + i] = kids[i]!
-        if (this.rec) {
-          for (let i = 0; i < kids.length; i++) this.code[head + 2 + kids.length + i] = this.followClass(d.parsers, i)
-        }
+        for (let i = 0; i < kids.length; i++) this.code[head + 2 + kids.length + i] = this.followClass(d.parsers, i)
         return head
       }
       case 'choice': {
@@ -1369,14 +1357,10 @@ class Encoder {
           : this.charClass(firstSetOf(d.parser, new Set(), rr, this.activeTrivia))
         const op = d.valueUnused ? OP_REPV : OP_REP
         const min = d.tag === 'many' ? 0 : d.min
-        // ip + 6 is the ITEM's expected set and ip + 7 the separator's sentinel
-        // class (−1: there is no separator). Both are only laid down for a
-        // recovery table, where the resync error's payload has to be the ITEM's
-        // set — `deriveExpected(combinator)` in repeat.ts:170, and
-        // `deriveExpectedArr([def.parser])` in codegen's emitMany.
-        return this.rec
-          ? this.emit(op, child, min, d.max ?? -1, -1, 0, this.expected(deriveExpected(d.parser)), itemClass)
-          : this.emit(op, child, min, d.max ?? -1, -1, 0)
+        // ip + 6 is the ITEM's expected set and ip + 7 the item's sentinel class:
+        // the resync error's payload is the ITEM's set (`deriveExpected` in
+        // `combinators/repeat.ts`).
+        return this.emit(op, child, min, d.max ?? -1, -1, 0, this.expected(deriveExpected(d.parser)), itemClass)
       }
       case 'sepBy': {
         const child = this.node(d.parser).ip
@@ -1393,20 +1377,15 @@ class Encoder {
         // it means the FIRST item failed, and that failure already set the
         // item's own set. So no committed grammar grows by a word.
         const flags = (d.trailing === 'allow' ? 1 : 0) | (d.keepSeparators === true ? 2 : 0) | (d.min >= 2 ? 4 : 0)
-        // A RECOVERY table always carries ip + 6 (the ITEM's expected set, which
+        // Every list row carries ip + 6 (the ITEM's expected set, which
         // the resync error reports) and ip + 7 (the SEPARATOR's sentinel class, so
         // a tolerant list resyncs to its own separator OR the enclosing
         // delimiter — `orSentinel(separator, term)` in repeat.ts:454 and
         // `_rec.or(sepSent, mySync)` in codegen's emitSepBy).
-        if (this.rec) {
-          return this.emit(
-            OP_REP, child, d.min, d.max ?? -1, sep, flags,
-            this.expected(deriveExpected(d.parser)), this.charClass(firstSetOf(d.separator, new Set(), this.refResolver(), this.activeTrivia)),
-          )
-        }
-        return d.min >= 2
-          ? this.emit(OP_REP, child, d.min, d.max ?? -1, sep, flags, this.expected(deriveExpected(d.parser)))
-          : this.emit(OP_REP, child, d.min, d.max ?? -1, sep, flags)
+        return this.emit(
+          OP_REP, child, d.min, d.max ?? -1, sep, flags,
+          this.expected(deriveExpected(d.parser)), this.charClass(firstSetOf(d.separator, new Set(), this.refResolver(), this.activeTrivia)),
+        )
       }
       case 'optional': {
         const head = this.emit(OP_OPT, this.lookaheadGate(d.parser, this.node(d.parser).ip))
@@ -1424,7 +1403,7 @@ class Encoder {
         const inner = d.parser._def as ParserDef
         if (inner.tag === 'sequence' && !this.memo.has(d.parser) && !this.pending.has(d.parser)) {
           const kids = inner.parsers.map(c => this.node(c).ip)
-          const head = this.emitHead(OP_SEQX, 2 + kids.length + (this.rec ? kids.length : 0))
+          const head = this.emitHead(OP_SEQX, 2 + 2 * kids.length)
           const projection = directArrayProjection(d.fn, d.fnSrc, kids.length)
           // A negative operand describes an already-parsed child (`~index`),
           // while every function-pool index remains non-negative.  The row is
@@ -1435,9 +1414,7 @@ class Encoder {
             : ~projection
           this.code[head + 2] = kids.length
           for (let i = 0; i < kids.length; i++) this.code[head + 3 + i] = kids[i]!
-          if (this.rec) {
-            for (let i = 0; i < kids.length; i++) this.code[head + 3 + kids.length + i] = this.followClass(inner.parsers, i)
-          }
+          for (let i = 0; i < kids.length; i++) this.code[head + 3 + kids.length + i] = this.followClass(inner.parsers, i)
           return head
         }
         const child = this.node(d.parser).ip
@@ -1938,7 +1915,7 @@ class Encoder {
       ...(this.settings.hostMode === undefined ? {} : { hostMode: this.settings.hostMode }),
       ...(this.runtimeOnly.size === 0 ? {} : { runtimeOnly: [...this.runtimeOnly].sort() }),
       ...(this.triviaSpecs.length === 0 ? {} : { triviaSpecs: this.triviaSpecs }),
-      ...(this.rec ? { rec: 1 as const } : {}),
+      rec: 1,
       ...(this.cov === undefined ? {} : { cov: this.cov }),
       lines: this.track ? 1 : 0,
     }, undefined, {

package/src/table/program.ts

@@ -213,7 +213,7 @@ export function covKindCode(kind: 'rule' | 'choice-arm' | 'dispatch-arm' | 'labe
   }
 }
 
-export function covKindName(code: 0 | 1 | 2 | 3): 'rule' | 'choice-arm' | 'dispatch-arm' | 'label' {
+function covKindName(code: 0 | 1 | 2 | 3): 'rule' | 'choice-arm' | 'dispatch-arm' | 'label' {
   switch (code) {
     case 0: return 'rule'
     case 1: return 'choice-arm'

package/src/table/scalar-terminal.ts

@@ -209,7 +209,7 @@ export function leadingScalarSequence(
  * choices, but only when EVERY arm yields literal authority. Longer literals
  * covered by a shorter prefix are removed (`@{-}` is already covered by `@{`).
  */
-export function leadingLiteralFamily(
+function leadingLiteralFamily(
   code: ArrayLike<number>, constants: readonly unknown[], ip: number,
   minDepth = 2, limit = 4,
 ): readonly number[] | undefined {

package/src/table/scan-shapes.ts

@@ -94,7 +94,7 @@ export type ScanShape =
   | { kind: 'alt'; arms: ScanShape[]; disjoint: boolean; firsts: Array<CharSet | null> }
 
 /** Backslash (`\`) code point — the escape lead char in string shapes. */
-export const BACKSLASH = 92
+const BACKSLASH = 92
 
 const META = new Set('()[]{}*+?|^$.'.split(''))
 
@@ -1137,15 +1137,6 @@ function caseFoldLiteralOrAlt(source: string): ScanShape | null {
   return { kind: 'alt', arms: litArms, disjoint: false, firsts: litArms.map(() => null) }
 }
 
-/**
- * `PARSEMAN_SCANTO=indexof` swaps the single-stop-char `until` scan loop for
- * `String.prototype.indexOf`. Defaults OFF so a baseline build is provably
- * byte-identical to the unmodified compiler, which is what makes the A/B a
- * toggle rather than two different compilers (docs/design/derived-tokenization.md
- * §16.3: the gate is a byte-identical parse tree against a toggled baseline).
- */
-const scanToIndexOfEnabled = (): boolean => process.env.PARSEMAN_SCANTO === 'indexof'
-
 export const classCond = (cVar: string, ranges: Array<[number, number]>): string =>
   ranges
     .map(([lo, hi]) => (lo === hi ? `${cVar} === ${lo}` : `(${cVar} >= ${lo} && ${cVar} <= ${hi})`))
@@ -1162,10 +1153,10 @@ export const litCond = (base: string, cps: number[], firstVar?: string): string
     .join(' && ')
 
 /** Line-terminator code points `.` does not match (`\n \r \u2028 \u2029`). */
-export const LINE_TERMINATORS = [10, 13, 8232, 8233] as const
+const LINE_TERMINATORS = [10, 13, 8232, 8233] as const
 
 /** The body-stop chars that abort a string match (excluded set minus quote/backslash). */
-export function stringHardStop(
+function stringHardStop(
   shape: Extract<ScanShape, { kind: 'string' }>,
 ): Array<[number, number]> {
   return shape.excluded.filter(
@@ -1467,24 +1458,12 @@ export function emitShapeMatch(
     const openChk = openLen === 1
       ? `${codeAt(start, firstChar)} === ${shape.open[0]}`
       : litCond(start, shape.open, firstChar)
-    // A SINGLE stop code point makes this scan a `memchr`, and V8 implements
-    // `String.prototype.indexOf` with a SIMD one. Measured 4.3x over the char
-    // loop on a 0.97 MiB corpus, 61 of 61 interleaved rounds
-    // (scratchpad/native-primitive-bakeoff/micro-scanto.mjs, regime (a)).
-    // Semantics are identical: the loop stops at the first stop char or at
-    // `input.length`, which is exactly `indexOf`'s hit / not-found pair.
-    // Multi-member stop sets stay on the loop — 3x indexOf + min measured at
-    // parity (0.992), because the min-reduce rescans the tail and discards it.
-    const stopCp =
-      shape.stop.length === 1 && shape.stop[0]![0] === shape.stop[0]![1] ? shape.stop[0]![0] : null
+    // A char loop, not `String.prototype.indexOf`. An opt-in `indexOf` variant
+    // (`PARSEMAN_SCANTO=indexof`) won 4.3x in a microbenchmark but was never
+    // claimed as a win on real grammars (CHANGELOG), and nothing enabled it; it
+    // last existed at `4a5d0341`.
     const scanLine =
-      stopCp !== null && stopCp <= 0xffff && scanToIndexOfEnabled()
-        ? // No temp is minted: a temp would renumber every later variable in the
-          // function and show up as an artifact-size delta that is counter
-          // renumbering rather than this change (the §9.1.4 / §9.2 effect).
-          `${ind}  ${j} = input.indexOf(${JSON.stringify(String.fromCharCode(stopCp))}, ${j})` +
-          `; if (${j} < 0) ${j} = input.length`
-        : `${ind}  while (${j} < input.length && !(${classCond(`input.charCodeAt(${j})`, shape.stop)})) ${j}++`
+      `${ind}  while (${j} < input.length && !(${classCond(`input.charCodeAt(${j})`, shape.stop)})) ${j}++`
     return {
       setup: [
         `${ind}let ${j} = ${start}`,
@@ -1549,23 +1528,13 @@ export function emitShapeMatch(
     ? `${codeAt(start, firstChar)} === ${shape.open[0]}`
     : litCond(start, shape.open, firstChar)
   const closeChk = litCond(j, shape.close)
-  // Scanning to a fixed close literal IS `indexOf`, and V8's is SIMD. The
-  // rewrite is exact: the loop stops at the first close literal at or after `j`,
-  // or runs out of room and leaves the token unterminated (no match). `indexOf`
-  // returns that same first index, or -1 for "never occurs" — mapped to
-  // `input.length`, which makes the `j + closeLen <= input.length` guard fail
-  // exactly where the loop's room check did. When it IS found the close literal
-  // matches by construction, so the guard's re-comparison is dropped.
-  const closeStr = String.fromCharCode(...shape.close)
-  const scanLines = scanToIndexOfEnabled()
-    ? [
-        `${ind}  ${j} = input.indexOf(${JSON.stringify(closeStr)}, ${j}); if (${j} < 0) ${j} = input.length`,
-        `${ind}  if (${j} + ${closeLen} <= input.length) ${endV} = ${j} + ${closeLen}`,
-      ]
-    : [
-        `${ind}  while (${j} + ${closeLen - 1} < input.length && !(${closeChk})) ${j}++`,
-        `${ind}  if (${j} + ${closeLen} <= input.length && (${closeChk})) ${endV} = ${j} + ${closeLen}`,
-      ]
+  // The loop stops at the first close literal at or after `j`, or runs out of
+  // room and leaves the token unterminated (no match). The same `indexOf`
+  // variant as `until` above was removed with it.
+  const scanLines = [
+    `${ind}  while (${j} + ${closeLen - 1} < input.length && !(${closeChk})) ${j}++`,
+    `${ind}  if (${j} + ${closeLen} <= input.length && (${closeChk})) ${endV} = ${j} + ${closeLen}`,
+  ]
   return {
     setup: [
       `${ind}let ${j} = ${start}`,

package/src/table/site-labels.ts

@@ -81,7 +81,7 @@ export const CAP_OFF = 1
 export const CAP_ON = 2
 
 /** Buffer raw-entry mode is not provable at this site. */
-export const RAW_UNKNOWN = 0
+const RAW_UNKNOWN = 0
 /** Buffer retains raw entry values. */
 export const RAW_CAPTURE = 1
 /** Buffer retains only a raw source-order count. */

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.59.0'
+export const PARSEMAN_VERSION = '0.59.1'