parseman 0.47.1 → 0.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/README.md +27 -21
  2. package/dist/analysis/diagnostics.js +2863 -353
  3. package/dist/analysis/diagnostics.js.map +3 -3
  4. package/dist/cli/index.js +2854 -344
  5. package/dist/cli/index.js.map +3 -3
  6. package/dist/combinators/expect.d.ts +1 -0
  7. package/dist/combinators/expect.d.ts.map +1 -1
  8. package/dist/combinators/expected.d.ts +8 -0
  9. package/dist/combinators/expected.d.ts.map +1 -0
  10. package/dist/combinators/first-set.d.ts +8 -0
  11. package/dist/combinators/first-set.d.ts.map +1 -1
  12. package/dist/combinators/grammar.d.ts.map +1 -1
  13. package/dist/combinators/keywords.d.ts +4 -0
  14. package/dist/combinators/keywords.d.ts.map +1 -1
  15. package/dist/combinators/literal.d.ts.map +1 -1
  16. package/dist/combinators/not.d.ts.map +1 -1
  17. package/dist/combinators/peek.d.ts.map +1 -1
  18. package/dist/combinators/regex.d.ts.map +1 -1
  19. package/dist/combinators/repeat.d.ts.map +1 -1
  20. package/dist/combinators/sequence.d.ts.map +1 -1
  21. package/dist/combinators/trivia-skip.d.ts +16 -3
  22. package/dist/combinators/trivia-skip.d.ts.map +1 -1
  23. package/dist/compiler/direct-projection.d.ts +12 -0
  24. package/dist/compiler/direct-projection.d.ts.map +1 -0
  25. package/dist/compiler/token-alphabet.d.ts +845 -5
  26. package/dist/compiler/token-alphabet.d.ts.map +1 -1
  27. package/dist/compiler/token-capability.d.ts +3 -0
  28. package/dist/compiler/token-capability.d.ts.map +1 -0
  29. package/dist/compiler/token-capability.js +3192 -0
  30. package/dist/compiler/token-capability.js.map +6 -0
  31. package/dist/cst/trivia-charscan.d.ts +3 -0
  32. package/dist/cst/trivia-charscan.d.ts.map +1 -1
  33. package/dist/cst/trivia-css-scanner.d.ts +5 -0
  34. package/dist/cst/trivia-css-scanner.d.ts.map +1 -0
  35. package/dist/cst/trivia-kinds.d.ts.map +1 -1
  36. package/dist/index.js +2862 -339
  37. package/dist/index.js.map +3 -3
  38. package/dist/language-service/index.js +5 -1
  39. package/dist/language-service/index.js.map +1 -1
  40. package/dist/plugin/index.js +2870 -347
  41. package/dist/plugin/index.js.map +3 -3
  42. package/dist/spec/index.js +315 -46
  43. package/dist/spec/index.js.map +2 -2
  44. package/dist/table/assemble.d.ts.map +1 -1
  45. package/dist/table/child-slots.d.ts.map +1 -1
  46. package/dist/table/compile-rule-map.d.ts +3 -1
  47. package/dist/table/compile-rule-map.d.ts.map +1 -1
  48. package/dist/table/emit-assembly.d.ts +12 -13
  49. package/dist/table/emit-assembly.d.ts.map +1 -1
  50. package/dist/table/emit.d.ts.map +1 -1
  51. package/dist/table/encode.d.ts.map +1 -1
  52. package/dist/table/exec-baseline.d.ts.map +1 -1
  53. package/dist/table/exec.d.ts.map +1 -1
  54. package/dist/table/index.js +2819 -300
  55. package/dist/table/index.js.map +3 -3
  56. package/dist/table/lex-body.d.ts +9 -0
  57. package/dist/table/lex-body.d.ts.map +1 -0
  58. package/dist/table/lex-program.d.ts +25 -0
  59. package/dist/table/lex-program.d.ts.map +1 -0
  60. package/dist/table/ops.d.ts +43 -7
  61. package/dist/table/ops.d.ts.map +1 -1
  62. package/dist/table/program.d.ts +50 -2
  63. package/dist/table/program.d.ts.map +1 -1
  64. package/dist/table/scalar-terminal.d.ts +13 -0
  65. package/dist/table/scalar-terminal.d.ts.map +1 -0
  66. package/dist/table/site-labels.d.ts.map +1 -1
  67. package/dist/types.d.ts +2 -0
  68. package/dist/types.d.ts.map +1 -1
  69. package/dist/version.d.ts +1 -1
  70. package/package.json +11 -11
  71. package/src/combinators/expect.ts +1 -0
  72. package/src/combinators/expected.ts +17 -0
  73. package/src/combinators/first-set.ts +58 -5
  74. package/src/combinators/grammar.ts +8 -3
  75. package/src/combinators/keywords.ts +22 -6
  76. package/src/combinators/literal.ts +2 -1
  77. package/src/combinators/not.ts +5 -1
  78. package/src/combinators/peek.ts +5 -1
  79. package/src/combinators/regex.ts +5 -3
  80. package/src/combinators/repeat.ts +4 -10
  81. package/src/combinators/sequence.ts +17 -14
  82. package/src/combinators/trivia-skip.ts +46 -13
  83. package/src/compiler/direct-projection.ts +43 -0
  84. package/src/compiler/token-alphabet.ts +4316 -12
  85. package/src/compiler/token-capability.ts +6 -0
  86. package/src/cst/trivia-charscan.ts +34 -0
  87. package/src/cst/trivia-css-scanner.ts +142 -0
  88. package/src/cst/trivia-kinds.ts +3 -1
  89. package/src/plugin/index.ts +4 -4
  90. package/src/table/assemble.ts +1211 -221
  91. package/src/table/child-slots.ts +2 -1
  92. package/src/table/compile-rule-map.ts +21 -3
  93. package/src/table/emit-assembly.ts +328 -85
  94. package/src/table/emit.ts +16 -2
  95. package/src/table/encode.ts +242 -50
  96. package/src/table/exec-baseline.ts +3 -7
  97. package/src/table/exec.ts +121 -7
  98. package/src/table/lex-body.ts +30 -0
  99. package/src/table/lex-program.ts +422 -0
  100. package/src/table/ops.ts +45 -7
  101. package/src/table/program.ts +46 -2
  102. package/src/table/scalar-terminal.ts +51 -0
  103. package/src/table/site-labels.ts +11 -1
  104. package/src/table/stamp.ts +1 -1
  105. package/src/types.ts +1 -1
  106. package/src/version.ts +1 -1
  107. package/dist/analysis/diagnostics.cjs +0 -13824
  108. package/dist/analysis/diagnostics.cjs.map +0 -6
  109. package/dist/cli/index.cjs +0 -14925
  110. package/dist/cli/index.cjs.map +0 -6
  111. package/dist/index.cjs +0 -16391
  112. package/dist/index.cjs.map +0 -6
  113. package/dist/language-service/index.cjs +0 -1304
  114. package/dist/language-service/index.cjs.map +0 -6
  115. package/dist/oracle/index.cjs +0 -254
  116. package/dist/oracle/index.cjs.map +0 -6
  117. package/dist/plugin/index.cjs +0 -16097
  118. package/dist/plugin/index.cjs.map +0 -6
  119. package/dist/run/index.cjs +0 -577
  120. package/dist/run/index.cjs.map +0 -6
  121. package/dist/spec/index.cjs +0 -6236
  122. package/dist/spec/index.cjs.map +0 -6
  123. package/dist/table/index.cjs +0 -11463
  124. package/dist/table/index.cjs.map +0 -6
@@ -1,9 +1,19 @@
1
1
  /**
2
- * Derived tokenization, part 1: the terminal ALPHABET.
2
+ * Derived tokenization groundwork: primitive kernels and lexical-token families.
3
3
  *
4
- * Parseman knows every terminal in the composed grammar at macro time — every
5
- * `literal`, `keywords`, and `regex`. This module walks the combinator graph
6
- * and assigns each distinct terminal a small integer TOKEN ID.
4
+ * `collectAlphabet()` is the preserved primitive-terminal scanner groundwork:
5
+ * it knows every `literal`, `keywords`, and `regex` in the composed grammar and
6
+ * assigns each one a small integer KERNEL id. Those ids are implementation
7
+ * details, not parser tokens. In particular, unwrapping
8
+ * `token(sequence(identifier, optional(literal('('))))` must never publish an
9
+ * identifier id followed by a `(` id: `token()` defines ONE contiguous source
10
+ * token and ONE CST leaf for the complete range.
11
+ *
12
+ * `collectLexicalAlphabet()` is the production-facing boundary. It assigns ids
13
+ * to authored `token()` families, normalizes effect-free bodies into canonical
14
+ * lexical IR, interns equivalent recognizers, and records the downstream
15
+ * `dispatch()` outcome classifiers over the same range. Primitive terminals
16
+ * stay private recognizer machinery.
7
17
  *
8
18
  * Two things are deliberately separated, because conflating them is what makes
9
19
  * a naive derived scanner useless:
@@ -20,7 +30,13 @@
20
30
  * Nothing here decides HOW a token is recognized (see `token-scanner.ts`); this
21
31
  * module only decides WHAT the tokens are and WHICH ones each site may see.
22
32
  */
23
- import type { Combinator, ParserDef } from '../types.ts'
33
+ import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts'
34
+ import { classifyFinalChoice, firstSetOf, intersects, matchesEmpty } from '../combinators/first-set.ts'
35
+ import { branchUsesRouted } from '../combinators/dispatch.ts'
36
+ import type { BalancedSpec } from '../combinators/scanTo.ts'
37
+ import { deriveExpected } from '../combinators/expect.ts'
38
+ import { assertionFailureExpected, directTerminalFailureExpected } from '../combinators/expected.ts'
39
+ import { keywordsRegExp } from '../combinators/keywords.ts'
24
40
 
25
41
  /** A terminal in the derived alphabet, with its globally assigned id. */
26
42
  export type TokenTerminal =
@@ -42,6 +58,1202 @@ export type Alphabet = {
42
58
  originOf: Map<number, Combinator<unknown>>
43
59
  }
44
60
 
61
+ /** Effect-free recognition IR underneath one authored `token()` boundary. */
62
+ export type LexicalIr =
63
+ | { readonly kind: 'literal'; readonly value: string; readonly caseInsensitive: boolean }
64
+ | { readonly kind: 'keywords'; readonly words: readonly string[]; readonly caseInsensitive: boolean; readonly boundary: string | undefined }
65
+ | { readonly kind: 'regex'; readonly source: string; readonly flags: string }
66
+ | { readonly kind: 'sequence'; readonly parts: readonly LexicalIr[] }
67
+ | { readonly kind: 'choice'; readonly arms: readonly LexicalIr[] }
68
+ | { readonly kind: 'repeat'; readonly body: LexicalIr; readonly min: number; readonly max: number | null; readonly greedy: boolean; readonly mode: 'possessive' | 'backtracking' }
69
+ | { readonly kind: 'assert'; readonly positive: boolean; readonly body: LexicalIr }
70
+ | {
71
+ readonly kind: 'balanced'
72
+ readonly open: string
73
+ readonly close: string
74
+ readonly strict: boolean
75
+ readonly raw: boolean
76
+ /** Ordered explicit skippers; ambient trivia/scanSkip precede them unless raw. */
77
+ readonly skip: readonly LexicalIr[]
78
+ }
79
+
80
+ /** First complete executable lexical body. It is intentionally a semantic
81
+ * shape rather than an opcode: the encoder relocates the regex into the table
82
+ * pool only after TOKEN wins the compiler-only cost comparison. */
83
+ export type OptionalSuffixLexBody = {
84
+ readonly kind: 'regex-optional-code-unit'
85
+ readonly source: string
86
+ readonly flags: string
87
+ readonly suffix: string
88
+ }
89
+
90
+ export type ExecutableOptionalSuffixLexBody = OptionalSuffixLexBody & {
91
+ /** Match the two authored terminal line-publication points exactly. */
92
+ readonly baseCanMatchNewline: boolean
93
+ readonly suffixCanMatchNewline: boolean
94
+ /** Failure authority belongs to the resolved final regex, not the lazy
95
+ * construction thunk that originally named it. */
96
+ readonly expected: readonly string[]
97
+ }
98
+
99
+ /** A direct authored terminal under token(). The boundary shell still owns
100
+ * source slicing/CST; this body owns exactly the terminal recognizer and lines. */
101
+ export type ExecutableTerminalLexBody = {
102
+ readonly kind: 'regex-terminal'
103
+ readonly source: string
104
+ readonly flags: string
105
+ readonly canMatchNewline: boolean
106
+ readonly expected: readonly string[]
107
+ }
108
+
109
+ export type ExecutableOrdered4LexBody = {
110
+ readonly kind: 'ordered4-terminals'
111
+ readonly commonFirstSet: FirstSet
112
+ readonly commonMissExpected: readonly string[]
113
+ readonly expected: readonly string[]
114
+ readonly arms: readonly [
115
+ ExecutableTerminalLexBody,
116
+ ExecutableTerminalLexBody,
117
+ ExecutableTerminalLexBody,
118
+ ExecutableTerminalLexBody,
119
+ ]
120
+ }
121
+
122
+ export type ExecutableNot2LexBody = {
123
+ readonly kind: 'not2-ordered4-terminal'
124
+ readonly guard0Expected: readonly string[]
125
+ readonly guard1Expected: readonly string[]
126
+ readonly terminalExpected: readonly string[]
127
+ readonly guard0: ExecutableOrdered4LexBody['arms']
128
+ readonly guard1: ExecutableTerminalLexBody
129
+ readonly terminal: ExecutableTerminalLexBody
130
+ }
131
+
132
+ export type ExecutableLexNode =
133
+ | { readonly kind: 'terminal'; readonly terminal: ExecutableTerminalLexBody }
134
+ | { readonly kind: 'sequence'; readonly parts: readonly ExecutableLexNode[] }
135
+ | {
136
+ readonly kind: 'choice'
137
+ readonly arms: readonly ExecutableLexNode[]
138
+ readonly firstSets: readonly FirstSet[]
139
+ readonly armExpected: readonly (readonly string[])[]
140
+ readonly expected: readonly string[]
141
+ }
142
+ | { readonly kind: 'not'; readonly body: ExecutableLexNode; readonly expected: readonly string[] }
143
+ | { readonly kind: 'optional'; readonly body: ExecutableLexNode }
144
+
145
+ export type ExecutableFixedLexBody = {
146
+ readonly kind: 'fixed-tree'
147
+ readonly root: ExecutableLexNode
148
+ }
149
+
150
+ /** A balanced token whose recognition remains owned by the canonical scanner
151
+ * pool. The selected shell removes only the losing token/parent topology; it
152
+ * never restates ambient scanSkip, recovery, or skipper semantics. */
153
+ export type ExecutableBalancedLexBody = {
154
+ readonly kind: 'balanced-scan'
155
+ readonly open: string
156
+ readonly close: string
157
+ readonly raw: boolean
158
+ readonly strict: boolean
159
+ readonly ownSkip: readonly Combinator<unknown>[]
160
+ }
161
+
162
+ export type ExecutableLexBody =
163
+ | ExecutableOptionalSuffixLexBody
164
+ | ExecutableTerminalLexBody
165
+ | ExecutableOrdered4LexBody
166
+ | ExecutableNot2LexBody
167
+ | ExecutableFixedLexBody
168
+ | ExecutableBalancedLexBody
169
+
170
+ export type LexBodyCandidate =
171
+ | { readonly strategy: 'character'; readonly estimatedOps: number }
172
+ | {
173
+ readonly strategy: 'token'
174
+ readonly estimatedOps: number
175
+ readonly body: OptionalSuffixLexBody
176
+ }
177
+
178
+ /** Exact recognizer shape admitted by the first end-to-end Stage-C tranche. */
179
+ export function optionalSuffixLexBody(ir: LexicalIr): OptionalSuffixLexBody | undefined {
180
+ if (ir.kind !== 'sequence' || ir.parts.length !== 2) return undefined
181
+ const [base, suffix] = ir.parts
182
+ if (base?.kind !== 'regex' || suffix?.kind !== 'repeat'
183
+ || suffix.min !== 0 || suffix.max !== 1 || !suffix.greedy
184
+ || suffix.mode !== 'possessive' || suffix.body.kind !== 'literal'
185
+ || suffix.body.caseInsensitive || suffix.body.value.length !== 1) return undefined
186
+ return {
187
+ kind: 'regex-optional-code-unit', source: base.source, flags: base.flags,
188
+ suffix: suffix.body.value,
189
+ }
190
+ }
191
+
192
+ /** Compare two COMPLETE candidates. The CHARACTER cost is the existing
193
+ * token-boundary + sequence + regex + optional + literal body. The TOKEN cost
194
+ * is one selected body and one materialization boundary; neither candidate is
195
+ * serialized until this decision returns. */
196
+ export function selectOptionalSuffixLexBody(ir: LexicalIr): LexBodyCandidate {
197
+ const body = optionalSuffixLexBody(ir)
198
+ const character: LexBodyCandidate = { strategy: 'character', estimatedOps: 5 }
199
+ if (body === undefined) return character
200
+ const token: LexBodyCandidate = { strategy: 'token', estimatedOps: 2, body }
201
+ return token.estimatedOps < character.estimatedOps ? token : character
202
+ }
203
+
204
+ /** Unwrap only scopes that token() has already installed identically. The
205
+ * returned parser is still the authored terminal/sequence; no effectful wrapper
206
+ * or final named winner is guessed from construction metadata. */
207
+ function directTokenBodyParser(
208
+ parser: Combinator<unknown>,
209
+ resolve?: (name: string) => Combinator<unknown> | undefined,
210
+ ): Combinator<unknown> | undefined {
211
+ const outer = parser._def
212
+ if (outer.tag !== 'token') return undefined
213
+ let body = outer.parser
214
+ const seen = new Set<Combinator<unknown>>()
215
+ while (!seen.has(body)) {
216
+ seen.add(body)
217
+ const wrapper = body._def
218
+ if (wrapper.tag === 'grammar') {
219
+ if (wrapper.clearTrivia !== true || wrapper.triviaParser !== undefined
220
+ || wrapper.captureTrivia === true || wrapper.rootCapture === 'opaque'
221
+ || wrapper.constructionTrackLines !== 'inherit'
222
+ || wrapper.constructionCaptureTriviaKinds !== undefined) return undefined
223
+ body = wrapper.parser
224
+ continue
225
+ }
226
+ if (wrapper.tag === 'lazy') {
227
+ const target = resolvedLazyTarget(body, resolve)
228
+ if (target === undefined) return undefined
229
+ body = target
230
+ continue
231
+ }
232
+ return body
233
+ }
234
+ return undefined
235
+ }
236
+
237
+ /** Refuse wrapper/effect lookalikes even when normalization erases them to the
238
+ * same language. This first executable body owns only the authored token
239
+ * boundary around a direct sequence(regex, optional(literal-one-code-unit)). */
240
+ export function directOptionalSuffixTokenBody(
241
+ parser: Combinator<unknown>,
242
+ resolve?: (name: string) => Combinator<unknown> | undefined,
243
+ ): ExecutableOptionalSuffixLexBody | undefined {
244
+ const inner = directTokenBodyParser(parser, resolve)
245
+ if (inner === undefined) return undefined
246
+ const sequence = inner._def
247
+ if (sequence.tag !== 'sequence' || sequence.parsers.length !== 2) return undefined
248
+ let baseParser = sequence.parsers[0]
249
+ const seenBase = new Set<Combinator<unknown>>()
250
+ while (baseParser !== undefined && baseParser._def.tag === 'lazy' && !seenBase.has(baseParser)) {
251
+ seenBase.add(baseParser)
252
+ baseParser = resolvedLazyTarget(baseParser, resolve)
253
+ }
254
+ const base = baseParser?._def
255
+ const optional = sequence.parsers[1]?._def
256
+ if (base?.tag !== 'regex' || optional?.tag !== 'optional') return undefined
257
+ const suffix = optional.parser._def
258
+ if (suffix.tag !== 'literal' || suffix.caseInsensitive
259
+ || suffix.value.length !== 1) return undefined
260
+ const selected = selectOptionalSuffixLexBody({
261
+ kind: 'sequence',
262
+ parts: [
263
+ { kind: 'regex', source: base.source, flags: base.flags },
264
+ {
265
+ kind: 'repeat',
266
+ body: { kind: 'literal', value: suffix.value, caseInsensitive: false },
267
+ min: 0, max: 1, greedy: true, mode: 'possessive',
268
+ },
269
+ ],
270
+ })
271
+ return selected.strategy === 'token' ? {
272
+ ...selected.body,
273
+ baseCanMatchNewline: baseParser!._meta.canMatchNewline,
274
+ suffixCanMatchNewline: optional.parser._meta.canMatchNewline,
275
+ expected: directTerminalFailureExpected(base),
276
+ } : undefined
277
+ }
278
+
279
+ /** Exact direct token terminal bodies. `keywords()` is represented by the same
280
+ * canonical sticky RegExp its authored parser and OP_RX lowering use. Literal
281
+ * stays CHARACTER for now: case-insensitive literal() intentionally uses ASCII
282
+ * folding, which is not JavaScript RegExp `/i` semantics for non-ASCII text. */
283
+ export function directTerminalTokenBody(
284
+ parser: Combinator<unknown>,
285
+ resolve?: (name: string) => Combinator<unknown> | undefined,
286
+ ): ExecutableTerminalLexBody | undefined {
287
+ const outer = parser._def
288
+ if (outer.tag !== 'token') return undefined
289
+ const body = directTokenBodyParser(parser, resolve)
290
+ if (body === undefined) return undefined
291
+ const def = body._def
292
+ if (def.tag === 'regex') return {
293
+ kind: 'regex-terminal', source: def.source, flags: def.flags,
294
+ canMatchNewline: body._meta.canMatchNewline,
295
+ expected: directTerminalFailureExpected(def),
296
+ }
297
+ if (def.tag !== 'keywords') return undefined
298
+ const re = keywordsRegExp(def.words, def.boundary, def.caseInsensitive)
299
+ return {
300
+ kind: 'regex-terminal', source: re.source, flags: re.flags,
301
+ canMatchNewline: body._meta.canMatchNewline,
302
+ expected: directTerminalFailureExpected(def),
303
+ }
304
+ }
305
+
306
+ function directTerminalParser(
307
+ parser: Combinator<unknown>,
308
+ resolve?: (name: string) => Combinator<unknown> | undefined,
309
+ ): Combinator<unknown> | undefined {
310
+ const seen = new Set<Combinator<unknown>>()
311
+ while (!seen.has(parser)) {
312
+ seen.add(parser)
313
+ const def = parser._def
314
+ if (def.tag === 'lazy') {
315
+ const target = resolvedLazyTarget(parser, resolve)
316
+ if (target === undefined) return undefined
317
+ parser = target
318
+ continue
319
+ }
320
+ if (def.tag === 'token') { parser = def.parser; continue }
321
+ // A grammar scope is not a recognition-only wrapper even when it clears
322
+ // trivia: it parses on a cloned context and owns line annotation plus
323
+ // failure-register publication. A childless composite body cannot erase
324
+ // that transaction. Keep the whole token on CHARACTER lowering instead.
325
+ if (def.tag === 'grammar') return undefined
326
+ return parser
327
+ }
328
+ return undefined
329
+ }
330
+
331
+ function directTerminalLexBody(
332
+ parser: Combinator<unknown>,
333
+ resolve?: (name: string) => Combinator<unknown> | undefined,
334
+ ): ExecutableTerminalLexBody | undefined {
335
+ const terminal = directTerminalParser(parser, resolve)
336
+ if (terminal === undefined) return undefined
337
+ const def = terminal._def
338
+ if (def.tag === 'literal') {
339
+ if (def.caseInsensitive) return undefined
340
+ const source = def.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
341
+ return {
342
+ kind: 'regex-terminal', source, flags: '',
343
+ canMatchNewline: terminal._meta.canMatchNewline,
344
+ expected: directTerminalFailureExpected(def),
345
+ }
346
+ }
347
+ if (def.tag === 'regex') return {
348
+ kind: 'regex-terminal', source: def.source, flags: def.flags,
349
+ canMatchNewline: terminal._meta.canMatchNewline,
350
+ expected: directTerminalFailureExpected(def),
351
+ }
352
+ if (def.tag !== 'keywords') return undefined
353
+ const re = keywordsRegExp(def.words, def.boundary, def.caseInsensitive)
354
+ return {
355
+ kind: 'regex-terminal', source: re.source, flags: re.flags,
356
+ canMatchNewline: terminal._meta.canMatchNewline,
357
+ expected: directTerminalFailureExpected(def),
358
+ }
359
+ }
360
+
361
+ /** First composite selected body: the hot Less at-keyword choice. Keep this a
362
+ * fixed template, not a recursive lexical interpreter. Equal finite first sets
363
+ * are load-bearing: at a matching lead all four arms really run in source
364
+ * order, while an excluded lead contributes only the choice's static opener
365
+ * set and no probe event. */
366
+ export function directOrdered4TokenBody(
367
+ parser: Combinator<unknown>,
368
+ resolve?: (name: string) => Combinator<unknown> | undefined,
369
+ ): ExecutableOrdered4LexBody | undefined {
370
+ const body = directTokenBodyParser(parser, resolve)
371
+ return body === undefined ? undefined : directOrdered4Body(body, resolve)
372
+ }
373
+
374
+ function directOrdered4Body(
375
+ body: Combinator<unknown>,
376
+ resolve?: (name: string) => Combinator<unknown> | undefined,
377
+ ): ExecutableOrdered4LexBody | undefined {
378
+ if (body === undefined) return undefined
379
+ const choice = body._def
380
+ if (choice.tag !== 'choice' || choice.parsers.length !== 4
381
+ || choice.strategy.tag !== 'firstMatch'
382
+ || choice.gates.some(gate => gate !== null)
383
+ || choice.autoNot.some(check => check !== null)) return undefined
384
+
385
+ const arms: ExecutableTerminalLexBody[] = []
386
+ let common: FirstSet | undefined
387
+ for (const arm of choice.parsers) {
388
+ const selected = directTerminalLexBody(arm, resolve)
389
+ if (selected === undefined) return undefined
390
+ if (selected.canMatchNewline) return undefined
391
+ const terminal = directTerminalParser(arm, resolve)!
392
+ const first = firstSetOf(terminal, new Set(), resolve)
393
+ if (first.kind !== 'ranges' || first.ranges.length === 0) return undefined
394
+ if (common === undefined) common = first
395
+ else if (JSON.stringify(common) !== JSON.stringify(first)) return undefined
396
+ arms.push(selected)
397
+ }
398
+ return {
399
+ kind: 'ordered4-terminals', commonFirstSet: common!,
400
+ commonMissExpected: deriveExpected(body),
401
+ expected: arms.flatMap(arm => arm.expected),
402
+ arms: arms as unknown as ExecutableOrdered4LexBody['arms'],
403
+ }
404
+ }
405
+
406
+ export function directNot2TokenBody(
407
+ parser: Combinator<unknown>,
408
+ resolve?: (name: string) => Combinator<unknown> | undefined,
409
+ ): ExecutableNot2LexBody | undefined {
410
+ const body = directTokenBodyParser(parser, resolve)
411
+ const sequence = body?._def
412
+ if (sequence?.tag !== 'sequence' || sequence.parsers.length !== 3) return undefined
413
+ const [guard0Parser, guard1Parser, terminalParser] = sequence.parsers
414
+ const guard0Def = guard0Parser?._def
415
+ const guard1Def = guard1Parser?._def
416
+ if (guard0Def?.tag !== 'not' || guard1Def?.tag !== 'not') return undefined
417
+ const ordered = directTokenBodyParser(guard0Def.parser, resolve)
418
+ ?? directTerminalParser(guard0Def.parser, resolve)
419
+ const guard0Body = ordered === undefined ? undefined : directOrdered4Body(ordered, resolve)
420
+ const guard1 = directTerminalLexBody(guard1Def.parser, resolve)
421
+ const terminal = terminalParser === undefined ? undefined : directTerminalLexBody(terminalParser, resolve)
422
+ if (guard0Body === undefined || guard1 === undefined || terminal === undefined
423
+ || guard1.canMatchNewline || terminal.canMatchNewline) return undefined
424
+ return {
425
+ kind: 'not2-ordered4-terminal',
426
+ guard0Expected: assertionFailureExpected(false, guard0Def.parser._tag),
427
+ guard1Expected: assertionFailureExpected(false, guard1Def.parser._tag),
428
+ terminalExpected: terminal.expected,
429
+ guard0: guard0Body.arms,
430
+ guard1, terminal,
431
+ }
432
+ }
433
+
434
+ function directFixedLexNode(
435
+ parser: Combinator<unknown>,
436
+ resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
437
+ active: Set<Combinator<unknown>>,
438
+ pureOnly = false,
439
+ ): ExecutableLexNode | undefined {
440
+ let current = parser
441
+ const wrappers = new Set<Combinator<unknown>>()
442
+ while (!wrappers.has(current)) {
443
+ wrappers.add(current)
444
+ const def = current._def
445
+ if (def.tag === 'lazy') {
446
+ const target = resolvedLazyTarget(current, resolve)
447
+ if (target === undefined) return undefined
448
+ current = target
449
+ continue
450
+ }
451
+ if (def.tag === 'token') { current = def.parser; continue }
452
+ break
453
+ }
454
+ if (wrappers.has(current) && current !== parser && current._def.tag === 'lazy') return undefined
455
+ if (active.has(current)) return undefined
456
+ active.add(current)
457
+ try {
458
+ const terminal = directTerminalLexBody(current, resolve)
459
+ if (terminal !== undefined) return { kind: 'terminal', terminal }
460
+ const def = current._def
461
+ if (def.tag === 'grammar') {
462
+ if (!pureOnly || (def.parser._def.tag !== 'token'
463
+ && (def.clearTrivia !== true || def.triviaParser !== undefined
464
+ || def.captureTrivia === true || def.rootCapture === 'opaque'))) return undefined
465
+ return directFixedLexNode(def.parser, resolve, active, true)
466
+ }
467
+ if (def.tag === 'sequence') {
468
+ if (def.parsers.length < 2 || def.parsers.length > 5) return undefined
469
+ const parts = def.parsers.map(part => directFixedLexNode(part, resolve, active, pureOnly))
470
+ if (parts.some(part => part === undefined)) return undefined
471
+ return { kind: 'sequence', parts: parts as ExecutableLexNode[] }
472
+ }
473
+ if (def.tag === 'choice') {
474
+ if ((def.parsers.length !== 2 && def.parsers.length !== 4 && def.parsers.length !== 8)
475
+ || def.strategy.tag !== 'firstMatch'
476
+ || def.gates.some(gate => gate !== null)
477
+ || def.autoNot.some(check => check !== null)) return undefined
478
+ const arms = def.parsers.map(arm => directFixedLexNode(arm, resolve, active, pureOnly))
479
+ if (arms.some(arm => arm === undefined)) return undefined
480
+ const final = classifyFinalChoice(def.parsers, resolve)
481
+ return {
482
+ kind: 'choice', arms: arms as ExecutableLexNode[],
483
+ firstSets: final.firstSets,
484
+ armExpected: def.parsers.map(deriveExpected),
485
+ expected: def.parsers.flatMap(deriveExpected),
486
+ }
487
+ }
488
+ if (def.tag === 'not') {
489
+ const body = directFixedLexNode(def.parser, resolve, active, true)
490
+ return body === undefined ? undefined : {
491
+ kind: 'not', body,
492
+ expected: assertionFailureExpected(false, def.parser._tag),
493
+ }
494
+ }
495
+ if (def.tag === 'optional') {
496
+ const body = directFixedLexNode(def.parser, resolve, active, pureOnly)
497
+ return body === undefined ? undefined : { kind: 'optional', body }
498
+ }
499
+ return undefined
500
+ } finally {
501
+ active.delete(current)
502
+ }
503
+ }
504
+
505
+ export function directFixedTokenBody(
506
+ parser: Combinator<unknown>,
507
+ resolve?: (name: string) => Combinator<unknown> | undefined,
508
+ ): ExecutableFixedLexBody | undefined {
509
+ const body = directTokenBodyParser(parser, resolve)
510
+ if (body === undefined) return undefined
511
+ const root = directFixedLexNode(body, resolve, new Set())
512
+ return root === undefined ? undefined : { kind: 'fixed-tree', root }
513
+ }
514
+
515
+ export function directBalancedTokenBody(
516
+ parser: Combinator<unknown>,
517
+ resolve?: (name: string) => Combinator<unknown> | undefined,
518
+ ): ExecutableBalancedLexBody | undefined {
519
+ if (directTokenBodyParser(parser, resolve) === undefined) return undefined
520
+ const spec = balancedSpecOf(parser)
521
+ return spec === undefined ? undefined : {
522
+ kind: 'balanced-scan', open: spec.open, close: spec.close,
523
+ raw: spec.raw, strict: spec.strict, ownSkip: spec.ownSkip,
524
+ }
525
+ }
526
+
527
+ export function directExecutableTokenBody(
528
+ parser: Combinator<unknown>,
529
+ resolve?: (name: string) => Combinator<unknown> | undefined,
530
+ ): ExecutableLexBody | undefined {
531
+ return directOptionalSuffixTokenBody(parser, resolve)
532
+ ?? directTerminalTokenBody(parser, resolve)
533
+ ?? directOrdered4TokenBody(parser, resolve)
534
+ ?? directNot2TokenBody(parser, resolve)
535
+ ?? directBalancedTokenBody(parser, resolve)
536
+ ?? directFixedTokenBody(parser, resolve)
537
+ }
538
+
539
+ /** One canonical recognizer spec, shared by every family with equal lexical IR. */
540
+ export type LexicalRecognizer = {
541
+ readonly id: number
542
+ readonly key: string
543
+ readonly ir: LexicalIr
544
+ /** Crosswalk into the compiler-wide canonical recognition-family interner. */
545
+ readonly capabilityFamilyId: number
546
+ }
547
+
548
+ /**
549
+ * Spelling-specific failure/probe contract. Families may share recognition IR
550
+ * without sharing where a structured spelling fails or what it expects.
551
+ */
552
+ export type LexicalDiagnosticPlan = {
553
+ readonly id: number
554
+ readonly body: Combinator<unknown>
555
+ }
556
+
557
+ /** Pointer-free, compiler-only checkpoints inside one authored token body. */
558
+ export type LexicalDiagnosticEvent =
559
+ | { readonly op: 'FAIL'; readonly state: number; readonly expectedId: number }
560
+ | {
561
+ readonly op: 'ASSERT'
562
+ readonly state: number
563
+ readonly innerStart: number
564
+ readonly innerEnd: number
565
+ readonly positive: boolean
566
+ readonly expectedId: number
567
+ readonly snapshotPolicy: 'saveLookaheadMark'
568
+ readonly execution: 'deferred'
569
+ }
570
+ | {
571
+ readonly op: 'REQUIRE'
572
+ readonly state: number
573
+ readonly childStart: number
574
+ readonly childEnd: number
575
+ readonly expectedId: number
576
+ readonly when: 'mandatory-iteration-2-to-min'
577
+ readonly committedChild: 'propagate'
578
+ readonly probe: 'child-only'
579
+ readonly anchor: 'repeat-position'
580
+ }
581
+ | {
582
+ readonly op: 'BAL_CLOSE_STRICT'
583
+ readonly state: number
584
+ readonly expectedId: number
585
+ readonly probe: true
586
+ readonly committed: false
587
+ readonly when: 'close-miss-after-open-body-success'
588
+ }
589
+ | {
590
+ readonly op: 'BAL_CLOSE_RECOVER'
591
+ readonly state: number
592
+ readonly expectedId: number
593
+ readonly probe: true
594
+ readonly committed: false
595
+ readonly when: 'close-miss-after-open-body-success'
596
+ readonly result: 'parseError'
597
+ readonly errorSpan: 'close-position'
598
+ readonly lineAnnotation: 'when-active'
599
+ readonly errorSink: 'when-active'
600
+ readonly cstErrorCapture: 'when-active'
601
+ }
602
+
603
+ /**
604
+ * Stage-B2 diagnostic metadata. It is intentionally absent from TableProgram:
605
+ * assertion execution and every token-boundary effect remain capability GAPs.
606
+ */
607
+ export type LexicalTransitionDiagnosticPlan = {
608
+ readonly id: number
609
+ readonly stateCount: number
610
+ readonly expected: readonly (readonly string[])[]
611
+ readonly events: readonly LexicalDiagnosticEvent[]
612
+ }
613
+
614
+ export type LexicalControlNode = {
615
+ readonly id: number
616
+ readonly parentControlId?: number
617
+ /** Authored parser kind, or the conditional/repeated balanced skipper role. */
618
+ readonly kind: ParserDef['tag'] | 'balanced' | 'balanced-skip'
619
+ readonly stateStart: number
620
+ readonly stateEnd: number
621
+ }
622
+
623
+ export type LexicalControlPlan = {
624
+ readonly id: number
625
+ /** Pointer-free source-control index over the one normalization session. */
626
+ readonly controls: readonly LexicalControlNode[]
627
+ }
628
+
629
+ /** One globally interned lexical family/base-token id. */
630
+ export type LexicalTokenFamily = {
631
+ readonly id: number
632
+ readonly recognizerId: number
633
+ }
634
+
635
+ /** One authored `token()` site, with spelling-specific diagnostics/effects. */
636
+ export type LexicalTokenSite = {
637
+ readonly parser: Combinator<unknown>
638
+ readonly body: Combinator<unknown>
639
+ /** Absent on a refused site; refused metadata allocates no diagnostic plan. */
640
+ readonly diagnosticId?: number
641
+ /** All three are absent when normalization declined. */
642
+ readonly familyId?: number
643
+ readonly recognizerId?: number
644
+ readonly refusal?: string
645
+ }
646
+
647
+ export type LexicalCapabilityStatus =
648
+ | { readonly kind: 'complete' }
649
+ | { readonly kind: 'gap'; readonly reason: string }
650
+ | { readonly kind: 'impossible'; readonly proof: string }
651
+
652
+ export type LexicalCapabilityPhase = {
653
+ readonly representation: LexicalCapabilityStatus
654
+ /** A complete body is constructible for every required reader/variant before selection. */
655
+ readonly executableLowering: LexicalCapabilityStatus
656
+ }
657
+
658
+ export type LexicalCapabilityObligations = {
659
+ readonly recognition: LexicalCapabilityPhase
660
+ readonly diagnostics: LexicalCapabilityPhase
661
+ readonly boundaryPlan: LexicalCapabilityPhase
662
+ readonly materializationPlan: LexicalCapabilityPhase
663
+ readonly supportedVariants: LexicalCapabilityPhase
664
+ readonly bindingAndReachability: LexicalCapabilityPhase
665
+ }
666
+
667
+ /** Built-in authored token() context transaction. Compiler-only in this tranche. */
668
+ export type LexicalBoundaryPlan = {
669
+ readonly id: 0
670
+ readonly kind: 'token-context-transaction'
671
+ }
672
+
673
+ /** Built-in authored token() source-range value/CST materialization. */
674
+ export type LexicalMaterializationPlan = {
675
+ readonly id: 0
676
+ readonly kind: 'token-source-range'
677
+ }
678
+
679
+ /** Exact construction-time grammar wrapper policy, not merely its effective context. */
680
+ export type LexicalGrammarWrapperSpec = {
681
+ readonly id: number
682
+ readonly sourceOperationId: number
683
+ readonly clearTrivia: boolean
684
+ readonly triviaBindingId?: number
685
+ readonly trackLines: 'on' | 'off' | 'inherit'
686
+ readonly captureTrivia: boolean
687
+ readonly captureTriviaKindsId?: number
688
+ readonly rootCapture: 'opaque' | 'inherit'
689
+ readonly clonePolicy: 'spread-existing-or-create-canonical'
690
+ readonly postChildPolicy: 'line-propagate-then-annotate-or-return'
691
+ }
692
+
693
+ /** Well-nested overlay over the sole LexicalIr/B2 control/state authority. */
694
+ export type LexicalWrapperFrame =
695
+ | {
696
+ readonly id: number
697
+ readonly parentFrameId?: number
698
+ readonly controlId: number
699
+ readonly stateStart: number
700
+ readonly stateEnd: number
701
+ readonly kind: 'token'
702
+ readonly boundaryPlanId: 0
703
+ readonly materializationPlanId: 0
704
+ }
705
+ | {
706
+ readonly id: number
707
+ readonly parentFrameId?: number
708
+ readonly controlId: number
709
+ readonly stateStart: number
710
+ readonly stateEnd: number
711
+ readonly kind: 'grammar'
712
+ readonly wrapperSpecId: number
713
+ }
714
+
715
+ export type LexicalBoundaryTopology = {
716
+ readonly id: number
717
+ /** Source-order outer-before-inner frames; recognition/control is not duplicated. */
718
+ readonly frames: readonly LexicalWrapperFrame[]
719
+ }
720
+
721
+ export type LexicalCapabilityContext = {
722
+ readonly trivia: Combinator<unknown> | undefined
723
+ readonly scanSkip: readonly Combinator<unknown>[]
724
+ readonly trackLines: boolean
725
+ readonly captureTrivia: boolean
726
+ readonly opaqueRootCapture: boolean
727
+ readonly dynamicState: boolean
728
+ }
729
+
730
+ /** Immutable compiler occurrence context. Live scanner callbacks are policy,
731
+ * never identity-bearing lexical facts or pending-range keys. */
732
+ export type LexicalContextSnapshot = {
733
+ readonly id: number
734
+ readonly key: string
735
+ readonly hasTrivia: boolean
736
+ readonly hasScanSkip: boolean
737
+ readonly trackLines: boolean
738
+ readonly captureTrivia: boolean
739
+ readonly opaqueRootCapture: boolean
740
+ readonly dynamicState: boolean
741
+ }
742
+
743
+ export type LexicalOwnSkipEntry = {
744
+ readonly semanticKey?: string
745
+ readonly ir?: LexicalIr
746
+ readonly status: LexicalCapabilityStatus
747
+ }
748
+
749
+ /** Construction-owned skipper children only. Ambient trivia/scanSkip never
750
+ * enter this pool because their object and `.parse` lookup are observable. */
751
+ export type LexicalOwnSkipPlan = {
752
+ readonly id: number
753
+ readonly entries: readonly LexicalOwnSkipEntry[]
754
+ }
755
+
756
+ export type LexicalScanConsumerPolicy =
757
+ | {
758
+ readonly id: number
759
+ readonly siteId: number
760
+ readonly path: string
761
+ readonly contextSnapshotId: number
762
+ readonly kind: 'scanTo'
763
+ readonly raw: boolean
764
+ readonly ambient: 'enumerate-trivia-then-scanSkip-per-parse-attempt' | 'none-raw'
765
+ readonly lookup: 'dynamic-parse-property-every-skip-test'
766
+ readonly context: 'detached-per-attempt-state-errors-only'
767
+ readonly ownSkipPlanId?: number
768
+ readonly pendingReuse: 'forbidden'
769
+ readonly status: LexicalCapabilityStatus
770
+ }
771
+ | {
772
+ readonly id: number
773
+ readonly siteId: number
774
+ readonly path: string
775
+ readonly contextSnapshotId: number
776
+ readonly kind: 'balanced'
777
+ readonly raw: boolean
778
+ readonly ambient: 'array-identity-interior-cache' | 'none-raw'
779
+ readonly cache: 'lookup-every-attempt-enumerate-on-miss'
780
+ readonly lookup: 'dynamic-parse-property-every-skip-test'
781
+ readonly context: 'token-cleared-original'
782
+ readonly ownSkipPlanId?: number
783
+ readonly pendingReuse: 'forbidden'
784
+ readonly status: LexicalCapabilityStatus
785
+ }
786
+
787
+ /**
788
+ * Compiler-only phase-A census record. These ids are deliberately not family
789
+ * ids: two authored sites can share one future language while remaining two
790
+ * independently auditable capability obligations.
791
+ */
792
+ export type LexicalCapabilitySite = {
793
+ readonly id: number
794
+ /** Final-graph compiler body identity. Never serialized as a parser pointer. */
795
+ readonly bodyId: number
796
+ /** Interned recognition-language summary; never used as a completeness id. */
797
+ readonly languageId: number
798
+ readonly path: string
799
+ /** Effective lexical scope at this occurrence; compiler-only, never a family id. */
800
+ readonly contextKey: string
801
+ readonly contextSnapshotId: number
802
+ /** Context seen by recognition after token() clears trivia/capture sinks. */
803
+ readonly recognitionContextKey: string
804
+ readonly context: LexicalCapabilityContext
805
+ readonly recognitionContext: LexicalCapabilityContext
806
+ readonly semanticKey: string
807
+ readonly atom: 'terminal' | 'token' | 'choice' | 'dispatch'
808
+ readonly parser: Combinator<unknown>
809
+ readonly obligations: LexicalCapabilityObligations
810
+ /** Internal body metadata only; never claims the boundary effects obligation. */
811
+ readonly diagnosticPlanId?: number
812
+ /** Compiler-only boundary overlay; absent when exact representation declined. */
813
+ readonly boundaryTopologyId?: number
814
+ /** Control ancestry authority used by the boundary overlay. */
815
+ readonly controlPlanId?: number
816
+ /** Scan consumers owned by this atomic occurrence, in structural order. */
817
+ readonly scanConsumerPolicyIds: readonly number[]
818
+ /** Derived from `obligations`; callers cannot independently set it. */
819
+ readonly status: LexicalCapabilityStatus
820
+ }
821
+
822
+ export type LexicalCapabilityLanguage = {
823
+ readonly id: number
824
+ readonly atom: LexicalCapabilitySite['atom']
825
+ readonly semanticKey: string
826
+ }
827
+
828
+ /**
829
+ * One compiler-wide canonical source-range language. IDs use the lexical
830
+ * family namespace even while phase B is disabled; decision plans and the
831
+ * legacy token collector cross-reference this one IR interner.
832
+ */
833
+ export type LexicalDecisionFamily = {
834
+ readonly id: number
835
+ readonly semanticKey: string
836
+ readonly ir: LexicalIr
837
+ }
838
+
839
+ /**
840
+ * A compatible view of one family range. `prefix` owns its own shorter end;
841
+ * consumers must never substitute the full family end (the `a | ab` rule).
842
+ */
843
+ export type LexicalDecisionOutcomeView =
844
+ | { readonly kind: 'whole'; readonly relation: 'equal' }
845
+ | {
846
+ readonly kind: 'predicate'
847
+ readonly relation: 'equal'
848
+ readonly match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>
849
+ }
850
+ | {
851
+ readonly kind: 'language'
852
+ readonly relation: 'equal' | 'prefix'
853
+ readonly ir: LexicalIr
854
+ }
855
+
856
+ /** Global, family-qualified atomic outcome/view identity. */
857
+ export type LexicalDecisionOutcome = {
858
+ readonly id: number
859
+ readonly familyId: number
860
+ readonly semanticKey: string
861
+ readonly view: LexicalDecisionOutcomeView
862
+ }
863
+
864
+ export type LexicalDecisionAcceptance =
865
+ | { readonly kind: 'outcomes'; readonly outcomeIds: readonly number[] }
866
+ | { readonly kind: 'otherwise'; readonly excludingOutcomeIds: readonly number[] }
867
+ | { readonly kind: 'unrestricted' }
868
+ | { readonly kind: 'impossible' }
869
+ | { readonly kind: 'gap'; readonly reason: string }
870
+
871
+ /** One source-ordered arm/route for one candidate family at one occurrence. */
872
+ export type LexicalDecisionArm = {
873
+ readonly armId: number
874
+ readonly acceptance: LexicalDecisionAcceptance
875
+ readonly usesRouted: boolean
876
+ readonly dynamicGate: boolean
877
+ }
878
+
879
+ export type LexicalDecisionFamilyPlan = {
880
+ readonly familyId: number
881
+ readonly arms: readonly LexicalDecisionArm[]
882
+ }
883
+
884
+ /** Source-ordered decision regions within which a recognized range may remain
885
+ * live. A dynamic scanner is its own closed epoch: callbacks and ambient skip
886
+ * policy may mutate parse state, so no lexical fact may cross that arm in
887
+ * either direction. This is compiler evidence for one complete TOKEN body,
888
+ * never a request to replay the CHARACTER body. */
889
+ export type LexicalDecisionReuseEpoch = {
890
+ readonly armIds: readonly number[]
891
+ readonly pendingReuse: 'forbidden' | 'unproved'
892
+ readonly boundary: 'none' | 'dynamic-scan'
893
+ }
894
+
895
+ /** Occurrence-local ordered decision proof; never serialized in this tranche. */
896
+ export type LexicalDecisionSite = {
897
+ readonly siteId: number
898
+ readonly atom: 'choice' | 'dispatch'
899
+ readonly path: string
900
+ readonly contextKey: string
901
+ readonly contextSnapshotId: number
902
+ readonly families: readonly LexicalDecisionFamilyPlan[]
903
+ /** Dispatch-only source-order routed policy. This remains Stage-A authority;
904
+ * the effect projection references the site rather than copying route bits. */
905
+ readonly routeUsesRouted?: readonly boolean[]
906
+ /** Families not proven at this occurrence remain admitted through this
907
+ * conservative relation. It is an explicit TOKEN-body route, never a request
908
+ * to replay the character parser. */
909
+ readonly fallback: 'unrestricted'
910
+ /** Missing inclusion/partition proofs reduce pruning precision only. Every
911
+ * authored arm still runs its own exact replacement recognizer in PEG order. */
912
+ readonly precisionNotes: readonly string[]
913
+ /** Dynamic scanner callbacks are effectful and can never publish a reusable
914
+ * cross-arm lexical fact. All other sites remain unproved in this tranche. */
915
+ readonly pendingReuse: 'forbidden' | 'unproved'
916
+ readonly reuseEpochs: readonly LexicalDecisionReuseEpoch[]
917
+ }
918
+
919
+ /** Final winner-resolved fixed choice classes, shared with table encoding. */
920
+ export type LexicalChoiceClassAuthority = {
921
+ readonly id: number
922
+ readonly armIds: readonly number[]
923
+ readonly firstSets: readonly FirstSet[]
924
+ }
925
+
926
+ /** A non-authored execution order such as longest literal first. */
927
+ export type LexicalChoiceExecutionOrder = {
928
+ readonly id: number
929
+ readonly armIds: readonly number[]
930
+ }
931
+
932
+ /** Exact classified spelling to authored arm identity. */
933
+ export type LexicalClassifySpellingMap = {
934
+ readonly id: number
935
+ readonly entries: readonly { readonly spelling: string; readonly armId: number }[]
936
+ }
937
+
938
+ /** Transform callback identities applied to an already recognized spelling. */
939
+ export type LexicalClassifyProjectionPlan = {
940
+ readonly id: number
941
+ readonly entries: readonly {
942
+ readonly armId: number
943
+ readonly transformIds: readonly number[]
944
+ readonly transformSourceIds: readonly (number | -1)[]
945
+ readonly transformCount: number
946
+ }[]
947
+ }
948
+
949
+ export type LexicalExpectedAuthority = {
950
+ readonly id: number
951
+ readonly values: readonly string[]
952
+ }
953
+
954
+ /** Compiler-local callback identity; never a TableProgram or public wire value. */
955
+ export type LexicalDecisionCallbackAuthority = {
956
+ readonly id: number
957
+ readonly callback:
958
+ | ((state: unknown) => boolean)
959
+ | ((value: unknown, span: { start: number; end: number }) => unknown)
960
+ }
961
+
962
+ /** Named emitted projection for a callback; compiler-only until Stage C selects. */
963
+ export type LexicalDecisionSourceAuthority = {
964
+ readonly id: number
965
+ readonly source: string
966
+ }
967
+
968
+ /** Exact post-success rejection checks, interned without re-deriving them. */
969
+ export type LexicalAutoNotAuthority = {
970
+ readonly id: number
971
+ readonly checks: readonly AutoNotCheck[]
972
+ }
973
+
974
+ export type LexicalFinalDecisionAuthority =
975
+ | {
976
+ readonly id: number
977
+ readonly atom: 'choice'
978
+ readonly decisionSiteId: number
979
+ readonly authoredArmIds: readonly number[]
980
+ readonly mode:
981
+ | { readonly kind: 'ordered' }
982
+ | { readonly kind: 'exclusive'; readonly classAuthorityId: number }
983
+ | { readonly kind: 'longest'; readonly orderAuthorityId: number }
984
+ | {
985
+ readonly kind: 'classify'
986
+ readonly superArmId: number
987
+ readonly spellingMapAuthorityId: number
988
+ readonly projectionAuthorityId: number
989
+ }
990
+ /** Static total-miss set only for strategies that never consult dynamic gates. */
991
+ readonly staticMissExpectedAuthorityId?: number
992
+ }
993
+ | {
994
+ readonly id: number
995
+ readonly atom: 'dispatch'
996
+ readonly decisionSiteId: number
997
+ readonly selectorBindingEdgeId: number
998
+ readonly classifierAuthorityId: number
999
+ readonly orderedRouteIds: readonly number[]
1000
+ readonly noRouteExpectedAuthorityId: number
1001
+ }
1002
+
1003
+ export type LexicalDecisionChildEffect = {
1004
+ readonly id: number
1005
+ readonly decisionSiteId: number
1006
+ readonly armId: number
1007
+ readonly childBindingEdgeId: number
1008
+ readonly expectedAuthorityId: number
1009
+ readonly gateCallbackId?: number
1010
+ readonly gateSourceId?: number
1011
+ readonly autoNotAuthorityId?: number
1012
+ }
1013
+
1014
+ export type LexicalDecisionEffectProgram = {
1015
+ readonly id: number
1016
+ readonly decisionSiteId: number
1017
+ readonly finalDecisionAuthorityId: number
1018
+ readonly phase: LexicalCapabilityPhase
1019
+ readonly childEffectIds: readonly number[]
1020
+ /** Exact fixed-edge projections consumed by the three existing readers. */
1021
+ readonly childBindingProjectionIds: readonly number[]
1022
+ readonly referenceTemplateId: number
1023
+ readonly capturedTemplateId: number
1024
+ readonly namedTemplateId: number
1025
+ readonly readerMask: number
1026
+ readonly variantMask: number
1027
+ readonly semanticDigest: number
1028
+ readonly pendingRollbackPolicyId: 0
1029
+ readonly commitPolicyId: 0
1030
+ }
1031
+
1032
+ /** Compiler-only C2 pools. No member is a TableProgram or runtime wire field. */
1033
+ export type LexicalDecisionEffectInventory = {
1034
+ readonly finalDecisionAuthorities: readonly LexicalFinalDecisionAuthority[]
1035
+ readonly decisionEffects: readonly LexicalDecisionEffectProgram[]
1036
+ readonly decisionChildEffects: readonly LexicalDecisionChildEffect[]
1037
+ readonly choiceClassAuthorities: readonly LexicalChoiceClassAuthority[]
1038
+ readonly choiceExecutionOrders: readonly LexicalChoiceExecutionOrder[]
1039
+ readonly classifySpellingMaps: readonly LexicalClassifySpellingMap[]
1040
+ readonly classifyProjectionPlans: readonly LexicalClassifyProjectionPlan[]
1041
+ readonly decisionExpectedAuthorities: readonly LexicalExpectedAuthority[]
1042
+ readonly decisionCallbackAuthorities: readonly LexicalDecisionCallbackAuthority[]
1043
+ readonly decisionSourceAuthorities: readonly LexicalDecisionSourceAuthority[]
1044
+ readonly decisionAutoNotAuthorities: readonly LexicalAutoNotAuthority[]
1045
+ }
1046
+
1047
+ /** One fixed incoming/root edge whose direct linked-body candidates remain owed. */
1048
+ export type LexicalBindingEdge = {
1049
+ readonly id: number
1050
+ readonly path: string
1051
+ readonly contextKey: string
1052
+ /** Effective context at the child occurrence, after the parent boundary. */
1053
+ readonly childContextKey: string
1054
+ readonly contextSnapshotId: number
1055
+ readonly parentTag: ParserDef['tag'] | 'root'
1056
+ readonly childTag: ParserDef['tag']
1057
+ readonly projectionId?: number
1058
+ readonly status: LexicalCapabilityStatus
1059
+ }
1060
+
1061
+ /** Compiler-owned fixed-edge lowering. These are numeric construction facts,
1062
+ * not a parse-time micro-IR: each reader consumes only its own preselected
1063
+ * template and the common digest proves all three describe the same edge. */
1064
+ export type LexicalBindingProjection = {
1065
+ readonly id: number
1066
+ readonly edgeId: number
1067
+ readonly parentBodyId: number
1068
+ readonly childBodyId: number
1069
+ readonly childOrdinal: number
1070
+ readonly referenceOperand: number
1071
+ readonly capturedSlotId: number
1072
+ readonly capturedTemplateId: number
1073
+ readonly namedSymbolId: number
1074
+ readonly namedTemplateId: number
1075
+ /** bit 0 reference, bit 1 captured/CSP, bit 2 named/emitted. */
1076
+ readonly readerMask: number
1077
+ /** Strict, recovery, tracking, CST, probe, and coverage construction modes. */
1078
+ readonly variantMask: number
1079
+ readonly semanticDigest: number
1080
+ }
1081
+
1082
+ /** One executable primitive-terminal candidate plus every fixed incoming edge.
1083
+ * This remains compiler-only until Stage C selects a complete semantic body. */
1084
+ export type LexicalTerminalProjection = {
1085
+ readonly id: number
1086
+ readonly siteId: number
1087
+ readonly bodyId: number
1088
+ readonly languageId: number
1089
+ readonly referenceBodyId: number
1090
+ readonly capturedTemplateId: number
1091
+ readonly namedSymbolId: number
1092
+ readonly incomingBindingProjectionIds: readonly number[]
1093
+ readonly readerMask: number
1094
+ readonly variantMask: number
1095
+ readonly semanticDigest: number
1096
+ readonly status: LexicalCapabilityStatus
1097
+ }
1098
+
1099
+ export type LexicalOutcomeMatch =
1100
+ | { readonly kind: 'exact'; readonly values: readonly string[]; readonly caseInsensitive: boolean }
1101
+ | { readonly kind: 'startsWith' | 'endsWith'; readonly value: string; readonly caseInsensitive: boolean }
1102
+ | { readonly kind: 'matches'; readonly value: string; readonly flags: string; readonly caseInsensitive: boolean }
1103
+ | { readonly kind: 'otherwise'; readonly excluding: readonly Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[] }
1104
+
1105
+ /** A globally reusable classification id for one family/range predicate. */
1106
+ export type LexicalOutcomeSpec = {
1107
+ readonly id: number
1108
+ readonly familyId: number
1109
+ readonly match: LexicalOutcomeMatch
1110
+ }
1111
+
1112
+ /** One compatible view over a family range at a particular dispatch site. */
1113
+ export type LexicalTokenOutcome = {
1114
+ readonly id: number
1115
+ readonly match: LexicalOutcomeMatch
1116
+ }
1117
+
1118
+ /** One ordered dispatch arm. IDs classify; this route owns the branch/cut. */
1119
+ export type LexicalTokenRoute = {
1120
+ readonly index: number
1121
+ readonly kind: 'exact' | 'matcher' | 'otherwise'
1122
+ readonly acceptedIds: readonly number[]
1123
+ readonly matches: readonly LexicalOutcomeMatch[]
1124
+ readonly parser: Combinator<unknown>
1125
+ readonly usesRouted: boolean
1126
+ }
1127
+
1128
+ /** Dispatch outcomes stay site-local; ordered dispatch semantics are unchanged. */
1129
+ export type LexicalTokenClassifier = {
1130
+ readonly dispatch: Combinator<unknown>
1131
+ readonly familyId: number
1132
+ /** True when reaching token() crossed a wrapper whose effects must still run. */
1133
+ readonly selectorEffects: boolean
1134
+ /** Compatible range views, flattened and allowed to repeat an id by route. */
1135
+ readonly outcomes: readonly LexicalTokenOutcome[]
1136
+ /** Authoritative ordered route/cut identity. */
1137
+ readonly routes: readonly LexicalTokenRoute[]
1138
+ }
1139
+
1140
+ export type LexicalAlphabet = {
1141
+ /**
1142
+ * Compiler-only graph metadata. `diagnostics`, `sites`, `classifiers`, and
1143
+ * `familyIdOf` contain live combinators/Maps and are NOT TableProgram data.
1144
+ * A serializer must project the numeric/IR specs and relocate site references.
1145
+ */
1146
+ readonly recognizers: readonly LexicalRecognizer[]
1147
+ readonly diagnostics: readonly LexicalDiagnosticPlan[]
1148
+ readonly families: readonly LexicalTokenFamily[]
1149
+ readonly sites: readonly LexicalTokenSite[]
1150
+ readonly outcomes: readonly LexicalOutcomeSpec[]
1151
+ readonly classifiers: readonly LexicalTokenClassifier[]
1152
+ readonly familyIdOf: ReadonlyMap<Combinator<unknown>, number>
1153
+ /** Whole-final-grammar, ownership-aware phase-A capability census. */
1154
+ readonly capabilities: readonly LexicalCapabilitySite[]
1155
+ /** Interned language view only; completeness is always occurrence-based. */
1156
+ readonly capabilityLanguages: readonly LexicalCapabilityLanguage[]
1157
+ /** Every distinct fixed parent/root edge, independently GAP until linked. */
1158
+ readonly bindingEdges: readonly LexicalBindingEdge[]
1159
+ readonly bindingProjections: readonly LexicalBindingProjection[]
1160
+ readonly terminalProjections: readonly LexicalTerminalProjection[]
1161
+ /** Phase-A compatible range/arm algebra; compiler-only and occurrence-local. */
1162
+ readonly decisionFamilies: readonly LexicalDecisionFamily[]
1163
+ readonly decisionOutcomes: readonly LexicalDecisionOutcome[]
1164
+ readonly decisions: readonly LexicalDecisionSite[]
1165
+ readonly decisionEffectPlan: LexicalDecisionEffectInventory
1166
+ readonly transitionDiagnostics: readonly LexicalTransitionDiagnosticPlan[]
1167
+ readonly boundaryPlans: readonly LexicalBoundaryPlan[]
1168
+ readonly materializationPlans: readonly LexicalMaterializationPlan[]
1169
+ readonly grammarWrapperSpecs: readonly LexicalGrammarWrapperSpec[]
1170
+ readonly grammarCaptureTriviaKinds: readonly (readonly string[])[]
1171
+ readonly boundaryTopologies: readonly LexicalBoundaryTopology[]
1172
+ readonly controlPlans: readonly LexicalControlPlan[]
1173
+ readonly contextSnapshots: readonly LexicalContextSnapshot[]
1174
+ readonly ownSkipPlans: readonly LexicalOwnSkipPlan[]
1175
+ readonly scanConsumerPolicies: readonly LexicalScanConsumerPolicy[]
1176
+ /** False means phase B is forbidden for the entire program. */
1177
+ readonly capabilityComplete: boolean
1178
+ }
1179
+
1180
+ export type LexicalCapabilityInventory = Pick<
1181
+ LexicalAlphabet,
1182
+ 'capabilities' | 'capabilityLanguages' | 'bindingEdges'
1183
+ | 'bindingProjections' | 'terminalProjections'
1184
+ | 'decisionFamilies' | 'decisionOutcomes' | 'decisions' | 'decisionEffectPlan'
1185
+ | 'transitionDiagnostics'
1186
+ | 'boundaryPlans' | 'materializationPlans' | 'grammarWrapperSpecs'
1187
+ | 'grammarCaptureTriviaKinds' | 'boundaryTopologies'
1188
+ | 'controlPlans'
1189
+ | 'contextSnapshots' | 'ownSkipPlans' | 'scanConsumerPolicies'
1190
+ | 'capabilityComplete'
1191
+ >
1192
+
1193
+ /** Numeric-only artifact projection; `TableProgram` aliases this contract. */
1194
+ export type NumericLexicalPlan = {
1195
+ readonly recognizerOffsets: readonly number[]
1196
+ readonly recognizerData: readonly number[]
1197
+ readonly outcomeOffsets: readonly number[]
1198
+ readonly outcomeData: readonly number[]
1199
+ readonly tokenSites: readonly number[]
1200
+ readonly sites: readonly number[]
1201
+ readonly routes: readonly number[]
1202
+ readonly accepted: readonly number[]
1203
+ }
1204
+
1205
+ /** Lexical-family ids occupy their own published namespace. */
1206
+ export const FIRST_LEXICAL_FAMILY_ID = 3
1207
+
1208
+ const foldAscii = (value: string): string => value.replace(/[A-Z]/g, c => c.toLowerCase())
1209
+
1210
+ /** Canonical, parser-free identity for one range predicate. */
1211
+ export function canonicalLexicalOutcomeKey(match: LexicalOutcomeMatch): string {
1212
+ if (match.kind === 'exact') {
1213
+ const values = [...new Set(match.values
1214
+ .map(value => match.caseInsensitive ? foldAscii(value) : value))].sort()
1215
+ return JSON.stringify({ kind: match.kind, values, caseInsensitive: match.caseInsensitive })
1216
+ }
1217
+ if (match.kind === 'startsWith' || match.kind === 'endsWith') {
1218
+ return JSON.stringify({
1219
+ kind: match.kind,
1220
+ value: match.caseInsensitive ? foldAscii(match.value) : match.value,
1221
+ caseInsensitive: match.caseInsensitive,
1222
+ })
1223
+ }
1224
+ if (match.kind === 'matches') {
1225
+ const rawFlags = match.caseInsensitive && !match.flags.includes('i') ? `${match.flags}i` : match.flags
1226
+ const flags = new RegExp('', rawFlags.replace(/g/g, '')).flags
1227
+ return JSON.stringify({ kind: match.kind, value: match.value, flags })
1228
+ }
1229
+ if (match.kind !== 'otherwise') throw new Error('parseman: unknown lexical outcome predicate')
1230
+ const excluding = match.excluding
1231
+ const fixed = excluding.filter((entry): entry is Extract<LexicalOutcomeMatch, { kind: 'startsWith' | 'endsWith' }> =>
1232
+ entry.kind === 'startsWith' || entry.kind === 'endsWith')
1233
+ const fixedContainsExact = (
1234
+ exact: Extract<LexicalOutcomeMatch, { kind: 'exact' }>,
1235
+ candidate: Extract<LexicalOutcomeMatch, { kind: 'startsWith' | 'endsWith' }>,
1236
+ ): boolean => {
1237
+ if (exact.caseInsensitive && !candidate.caseInsensitive && /[A-Za-z]/.test(candidate.value)) return false
1238
+ const needle = candidate.caseInsensitive ? foldAscii(candidate.value) : candidate.value
1239
+ return exact.values.every(raw => {
1240
+ const value = candidate.caseInsensitive ? foldAscii(raw) : raw
1241
+ return candidate.kind === 'startsWith' ? value.startsWith(needle) : value.endsWith(needle)
1242
+ })
1243
+ }
1244
+ const reduced: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[] = []
1245
+ for (const entry of excluding) {
1246
+ if (entry.kind !== 'exact') { reduced.push(entry); continue }
1247
+ const values = entry.values.filter(value => !fixed.some(candidate =>
1248
+ fixedContainsExact({ ...entry, values: [value] }, candidate)))
1249
+ if (values.length > 0) reduced.push({ ...entry, values })
1250
+ }
1251
+ return JSON.stringify({
1252
+ kind: match.kind,
1253
+ excluding: [...new Set(reduced.map(canonicalLexicalOutcomeKey))].sort(),
1254
+ })
1255
+ }
1256
+
45
1257
  function keyOf(def: ParserDef): string | undefined {
46
1258
  switch (def.tag) {
47
1259
  case 'literal': return `L\u0000${def.value}\u0000${def.caseInsensitive ? 'i' : ''}`
@@ -60,6 +1272,44 @@ function terminalOf(def: ParserDef, id: number): TokenTerminal | undefined {
60
1272
  }
61
1273
  }
62
1274
 
1275
+ /** Does a final rule-map winner bottom out at the reference being resolved? */
1276
+ export function winnerWrapsReference(
1277
+ winner: Combinator<unknown>,
1278
+ reference: Combinator<unknown>,
1279
+ ): boolean {
1280
+ let current = winner
1281
+ const seen = new Set<Combinator<unknown>>()
1282
+ // Only wrappers introduced by rules()/parser() are transparent here. Walking
1283
+ // into a sequence or choice would mistake ordinary recursive use for an alias.
1284
+ while (!seen.has(current)) {
1285
+ if (current === reference) return true
1286
+ seen.add(current)
1287
+ const def = current._def
1288
+ if (def.tag !== 'grammar' && def.tag !== 'trivia') return false
1289
+ current = def.parser
1290
+ }
1291
+ return false
1292
+ }
1293
+
1294
+ /** Final winner resolution is authoritative; a stale construction thunk is fallback only. */
1295
+ function resolvedLazyTarget(
1296
+ parser: Combinator<unknown>,
1297
+ resolve?: (name: string) => Combinator<unknown> | undefined,
1298
+ ): Combinator<unknown> | undefined {
1299
+ const def = parser._def
1300
+ if (def.tag !== 'lazy') return undefined
1301
+ const name = (parser as unknown as { _ruleName?: string })._ruleName
1302
+ if (name !== undefined && resolve !== undefined) {
1303
+ const winner = resolve(name)
1304
+ // A named rule whose whole definition is itself a lazy alias is stamped
1305
+ // with its OWN name. Resolving that name returns the same object, not the
1306
+ // referenced winner; only a distinct winner supersedes the construction
1307
+ // thunk.
1308
+ if (winner !== undefined && !winnerWrapsReference(winner, parser)) return winner
1309
+ }
1310
+ try { return def.thunk() } catch { return undefined }
1311
+ }
1312
+
63
1313
  /** Direct sub-parsers of a def, resolving a lazy through `resolve` when its own thunk is a hole. */
64
1314
  export function tokenChildren(
65
1315
  p: Combinator<unknown>,
@@ -78,13 +1328,7 @@ export function tokenChildren(
78
1328
  if (Array.isArray(rec.cases)) for (const c of rec.cases as Array<{ parser: Combinator<unknown> }>) push(c.parser)
79
1329
  if (Array.isArray(rec.matchers)) for (const c of rec.matchers as Array<{ parser: Combinator<unknown> }>) push(c.parser)
80
1330
  if (d.tag === 'lazy') {
81
- let target: Combinator<unknown> | undefined
82
- try { target = d.thunk() } catch { target = undefined }
83
- if (target === undefined && resolve !== undefined) {
84
- const name = (p as unknown as { _ruleName?: string })._ruleName
85
- if (name !== undefined) target = resolve(name)
86
- }
87
- push(target)
1331
+ push(resolvedLazyTarget(p, resolve))
88
1332
  }
89
1333
  return out
90
1334
  }
@@ -121,6 +1365,3066 @@ export function collectAlphabet(
121
1365
  return { terminals, byKey, originOf }
122
1366
  }
123
1367
 
1368
+ type Normalized = { ir: LexicalIr } | { refusal: string }
1369
+
1370
+ type PendingLexicalWrapperFrame =
1371
+ | {
1372
+ readonly id: number
1373
+ readonly parentFrameId?: number
1374
+ readonly controlId: number
1375
+ readonly kind: 'token'
1376
+ stateStart: number
1377
+ stateEnd: number
1378
+ }
1379
+ | {
1380
+ readonly id: number
1381
+ readonly parentFrameId?: number
1382
+ readonly controlId: number
1383
+ readonly kind: 'grammar'
1384
+ readonly parser: Combinator<unknown>
1385
+ stateStart: number
1386
+ stateEnd: number
1387
+ }
1388
+
1389
+ type LexicalNormalizeSession = {
1390
+ nextState: number
1391
+ readonly expected: string[][]
1392
+ readonly expectedIds: Map<string, number>
1393
+ readonly events: LexicalDiagnosticEvent[]
1394
+ readonly wrapperFrames: PendingLexicalWrapperFrame[]
1395
+ readonly wrapperStack: number[]
1396
+ readonly controls: LexicalControlNode[]
1397
+ readonly controlStack: number[]
1398
+ }
1399
+
1400
+ function newNormalizeSession(): LexicalNormalizeSession {
1401
+ return {
1402
+ nextState: 0, expected: [], expectedIds: new Map(), events: [],
1403
+ wrapperFrames: [], wrapperStack: [], controls: [], controlStack: [],
1404
+ }
1405
+ }
1406
+
1407
+ function withLexicalControl(
1408
+ session: LexicalNormalizeSession | undefined,
1409
+ kind: LexicalControlNode['kind'],
1410
+ body: () => Normalized,
1411
+ ): Normalized {
1412
+ if (session === undefined) return body()
1413
+ const id = session.controls.length
1414
+ const parentControlId = session.controlStack[session.controlStack.length - 1]
1415
+ const node: LexicalControlNode = {
1416
+ id, ...(parentControlId === undefined ? {} : { parentControlId }), kind,
1417
+ stateStart: session.nextState, stateEnd: -1,
1418
+ }
1419
+ session.controls.push(node)
1420
+ session.controlStack.push(id)
1421
+ try {
1422
+ const result = body()
1423
+ ;(node as { stateEnd: number }).stateEnd = session.nextState
1424
+ return result
1425
+ } finally {
1426
+ const popped = session.controlStack.pop()
1427
+ if (popped !== id) throw new Error('parseman: lexical control stack is not well nested')
1428
+ }
1429
+ }
1430
+
1431
+ function withLexicalWrapper(
1432
+ session: LexicalNormalizeSession | undefined,
1433
+ kind: 'token' | 'grammar',
1434
+ parser: Combinator<unknown>,
1435
+ body: () => Normalized,
1436
+ ): Normalized {
1437
+ if (session === undefined) return body()
1438
+ const controlId = session.controlStack[session.controlStack.length - 1]
1439
+ if (controlId === undefined) throw new Error('parseman: lexical wrapper lacks a control anchor')
1440
+ const id = session.wrapperFrames.length
1441
+ const parentFrameId = session.wrapperStack[session.wrapperStack.length - 1]
1442
+ const common = {
1443
+ id, ...(parentFrameId === undefined ? {} : { parentFrameId }), controlId,
1444
+ stateStart: session.nextState, stateEnd: -1,
1445
+ }
1446
+ const frame: PendingLexicalWrapperFrame = kind === 'token'
1447
+ ? { ...common, kind }
1448
+ : { ...common, kind, parser }
1449
+ session.wrapperFrames.push(frame)
1450
+ session.wrapperStack.push(id)
1451
+ try {
1452
+ const result = body()
1453
+ frame.stateEnd = session.nextState
1454
+ return result
1455
+ } finally {
1456
+ const popped = session.wrapperStack.pop()
1457
+ if (popped !== id) throw new Error('parseman: lexical wrapper stack is not well nested')
1458
+ }
1459
+ }
1460
+
1461
+ function lexicalState(session?: LexicalNormalizeSession): number {
1462
+ return session === undefined ? -1 : session.nextState++
1463
+ }
1464
+
1465
+ function lexicalExpected(session: LexicalNormalizeSession, expected: readonly string[]): number {
1466
+ const key = JSON.stringify(expected)
1467
+ const prior = session.expectedIds.get(key)
1468
+ if (prior !== undefined) return prior
1469
+ const id = session.expected.length
1470
+ session.expectedIds.set(key, id)
1471
+ session.expected.push([...expected])
1472
+ return id
1473
+ }
1474
+
1475
+ function lexicalFail(session: LexicalNormalizeSession | undefined, state: number, expected: readonly string[]): void {
1476
+ if (session !== undefined) {
1477
+ session.events.push({ op: 'FAIL', state, expectedId: lexicalExpected(session, expected) })
1478
+ }
1479
+ }
1480
+
1481
+ function regexFlags(flags: string): string {
1482
+ // regex() itself owns stickiness and ignores authored global/sticky state.
1483
+ return new RegExp('', flags.replace(/[gy]/g, '')).flags
1484
+ }
1485
+
1486
+ function sequenceIr(parts: LexicalIr[]): LexicalIr {
1487
+ const flat: LexicalIr[] = []
1488
+ for (const part of parts) {
1489
+ if (part.kind === 'sequence') flat.push(...part.parts)
1490
+ else flat.push(part)
1491
+ }
1492
+ return flat.length === 1 ? flat[0]! : { kind: 'sequence', parts: flat }
1493
+ }
1494
+
1495
+ /**
1496
+ * The first production normalization is deliberately proof-bounded: a single
1497
+ * repeated positive character class followed by one optional escaped literal.
1498
+ * This covers the identifier + optional '(' family while keeping regex
1499
+ * backtracking distinct from PEG possessive repetition everywhere else.
1500
+ */
1501
+ function provenOptionalLiteralSuffix(
1502
+ source: string,
1503
+ flags: string,
1504
+ ): { base: string; suffix: string } | undefined {
1505
+ const match = /^(\[(?:\\.|[^\]\\])+\]\+)(\\[.*+?^$(){}|[\]\\/])\?$/.exec(source)
1506
+ if (match === null) return undefined
1507
+ const base = match[1]!
1508
+ const suffix = match[2]!.slice(1)
1509
+ // A one-character oracle is exact for this bounded base shape: the base is one
1510
+ // positive character class repeated with '+'. If the class cannot consume the
1511
+ // suffix character, the regex quantifier cannot backtrack across the suffix.
1512
+ const baseFlags = regexFlags(flags)
1513
+ if (new RegExp(`^(?:${base})$`, baseFlags).test(suffix)) return undefined
1514
+ return { base, suffix }
1515
+ }
1516
+
1517
+ function normalizeRegex(source: string, flags: string, session?: LexicalNormalizeSession): LexicalIr {
1518
+ const failure = directTerminalFailureExpected({ tag: 'regex', source, flags })
1519
+ const proof = provenOptionalLiteralSuffix(source, flags)
1520
+ if (proof === undefined) {
1521
+ const state = lexicalState(session)
1522
+ lexicalFail(session, state, failure)
1523
+ return { kind: 'regex', source, flags }
1524
+ }
1525
+ lexicalState(session) // synthetic sequence
1526
+ const baseState = lexicalState(session)
1527
+ lexicalFail(session, baseState, failure)
1528
+ lexicalState(session) // synthetic optional repeat
1529
+ lexicalState(session) // synthetic optional literal; no invented FAIL event
1530
+ return {
1531
+ kind: 'sequence',
1532
+ parts: [
1533
+ { kind: 'regex', source: proof.base, flags },
1534
+ {
1535
+ kind: 'repeat',
1536
+ body: {
1537
+ kind: 'literal',
1538
+ value: proof.suffix,
1539
+ caseInsensitive: flags.includes('i') && /[A-Za-z]/.test(proof.suffix),
1540
+ },
1541
+ min: 0,
1542
+ max: 1,
1543
+ greedy: true,
1544
+ mode: 'possessive',
1545
+ },
1546
+ ],
1547
+ }
1548
+ }
1549
+
1550
+ function normalizeLexical(
1551
+ parser: Combinator<unknown>,
1552
+ resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
1553
+ stack: Set<Combinator<unknown>>,
1554
+ session?: LexicalNormalizeSession,
1555
+ ): Normalized {
1556
+ if (stack.has(parser)) return { refusal: 'recursive token body' }
1557
+ stack.add(parser)
1558
+ try {
1559
+ return withLexicalControl(session, parser._def.tag, () =>
1560
+ normalizeLexicalBody(parser, resolve, stack, session))
1561
+ } finally {
1562
+ stack.delete(parser)
1563
+ }
1564
+ }
1565
+
1566
+ function normalizeLexicalBody(
1567
+ parser: Combinator<unknown>,
1568
+ resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
1569
+ stack: Set<Combinator<unknown>>,
1570
+ session?: LexicalNormalizeSession,
1571
+ ): Normalized {
1572
+ const done = (result: Normalized): Normalized => result
1573
+ const child = (value: Combinator<unknown>): Normalized => normalizeLexical(value, resolve, stack, session)
1574
+ const def = parser._def
1575
+
1576
+ switch (def.tag) {
1577
+ case 'literal': {
1578
+ const state = lexicalState(session)
1579
+ lexicalFail(session, state, directTerminalFailureExpected(def))
1580
+ return done({ ir: {
1581
+ kind: 'literal',
1582
+ value: def.caseInsensitive ? def.value.replace(/[A-Z]/g, c => c.toLowerCase()) : def.value,
1583
+ caseInsensitive: def.caseInsensitive,
1584
+ } })
1585
+ }
1586
+ case 'keywords': {
1587
+ const state = lexicalState(session)
1588
+ lexicalFail(session, state, directTerminalFailureExpected(def))
1589
+ const words = [...new Set(def.words.map(word => def.caseInsensitive
1590
+ ? word.replace(/[A-Z]/g, c => c.toLowerCase())
1591
+ : word))].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0))
1592
+ return done({ ir: {
1593
+ kind: 'keywords', words, caseInsensitive: def.caseInsensitive, boundary: def.boundary,
1594
+ } })
1595
+ }
1596
+ case 'regex':
1597
+ return done({ ir: normalizeRegex(def.source, regexFlags(def.flags), session) })
1598
+ case 'sequence': {
1599
+ lexicalState(session)
1600
+ const parts: LexicalIr[] = []
1601
+ for (const entry of def.parsers) {
1602
+ const normalized = child(entry)
1603
+ if ('refusal' in normalized) return done(normalized)
1604
+ if (normalized.ir.kind === 'sequence') parts.push(...normalized.ir.parts)
1605
+ else parts.push(normalized.ir)
1606
+ }
1607
+ return done({ ir: sequenceIr(parts) })
1608
+ }
1609
+ case 'choice': {
1610
+ lexicalState(session)
1611
+ if (def.gates.some(gate => gate !== null)) return done({ refusal: 'choice has a dynamic gate' })
1612
+ if (def.autoNot.some(check => check !== null)) return done({ refusal: 'choice has contextual prefix rejection' })
1613
+ if (!def.disjoint && def.strategy.tag !== 'firstMatch') {
1614
+ return done({ refusal: `choice strategy ${def.strategy.tag} is not canonical lexical order` })
1615
+ }
1616
+ const arms: LexicalIr[] = []
1617
+ for (const entry of def.parsers) {
1618
+ const normalized = child(entry)
1619
+ if ('refusal' in normalized) return done(normalized)
1620
+ arms.push(normalized.ir)
1621
+ }
1622
+ return done({ ir: arms.length === 1 ? arms[0]! : { kind: 'choice', arms } })
1623
+ }
1624
+ case 'optional': {
1625
+ lexicalState(session)
1626
+ const normalized = child(def.parser)
1627
+ return done('refusal' in normalized ? normalized : {
1628
+ ir: { kind: 'repeat', body: normalized.ir, min: 0, max: 1, greedy: true, mode: 'possessive' },
1629
+ })
1630
+ }
1631
+ case 'many':
1632
+ case 'oneOrMore': {
1633
+ const state = lexicalState(session)
1634
+ if (matchesEmpty(def.parser)) return done({ refusal: 'repeat body may not make progress' })
1635
+ const childStart = session?.nextState ?? -1
1636
+ const normalized = child(def.parser)
1637
+ if ('refusal' in normalized) return done(normalized)
1638
+ if (session !== undefined && def.min > 1) {
1639
+ const expected = deriveExpected(def.parser)
1640
+ session.events.push({
1641
+ op: 'REQUIRE', state, childStart, childEnd: session.nextState,
1642
+ expectedId: lexicalExpected(session, expected.length > 0 ? expected : [def.parser._tag]),
1643
+ when: 'mandatory-iteration-2-to-min', committedChild: 'propagate',
1644
+ probe: 'child-only', anchor: 'repeat-position',
1645
+ })
1646
+ }
1647
+ return done({
1648
+ ir: { kind: 'repeat', body: normalized.ir, min: def.min, max: def.max ?? null, greedy: true, mode: 'possessive' },
1649
+ })
1650
+ }
1651
+ case 'not':
1652
+ case 'peek': {
1653
+ const state = lexicalState(session)
1654
+ const innerStart = session?.nextState ?? -1
1655
+ const normalized = child(def.parser)
1656
+ if ('refusal' in normalized) return done(normalized)
1657
+ if (session !== undefined) session.events.push({
1658
+ op: 'ASSERT', state, innerStart, innerEnd: session.nextState,
1659
+ positive: def.tag === 'peek',
1660
+ expectedId: lexicalExpected(session, assertionFailureExpected(def.tag === 'peek', def.parser._tag)),
1661
+ snapshotPolicy: 'saveLookaheadMark', execution: 'deferred',
1662
+ })
1663
+ return done({
1664
+ ir: { kind: 'assert', positive: def.tag === 'peek', body: normalized.ir },
1665
+ })
1666
+ }
1667
+ case 'lazy': {
1668
+ const target = resolvedLazyTarget(parser, resolve)
1669
+ return done(target === undefined ? { refusal: 'unresolved lazy token body' } : child(target))
1670
+ }
1671
+ case 'token':
1672
+ return done(withLexicalWrapper(session, 'token', parser, () => child(def.parser)))
1673
+ case 'trivia':
1674
+ return done(child(def.parser))
1675
+ case 'grammar':
1676
+ // A direct token child installs its own contiguous lexical context before
1677
+ // its body runs. The enclosing scope's trivia/capture/line policy is
1678
+ // therefore recognition-inert (its observable effects remain a separate
1679
+ // capability obligation). Keep every other trivia-bearing scope distinct:
1680
+ // trivia between ordinary child terms changes the accepted language.
1681
+ if (def.parser._def.tag === 'token') {
1682
+ return done(withLexicalWrapper(session, 'grammar', parser, () => child(def.parser)))
1683
+ }
1684
+ if (def.trackLines) return done({ refusal: 'token body has line-tracking effects' })
1685
+ if (def.triviaParser !== undefined || def.captureTrivia === true || def.rootCapture !== undefined) {
1686
+ return done({ refusal: 'token body has a trivia-bearing scope' })
1687
+ }
1688
+ return done(withLexicalWrapper(session, 'grammar', parser, () => child(def.parser)))
1689
+ case 'transform': return done({ refusal: 'transform is effectful' })
1690
+ case 'leaf': return done({ refusal: 'leaf is effectful' })
1691
+ case 'node': return done({ refusal: 'node is effectful' })
1692
+ case 'field': return done({ refusal: 'field is effectful' })
1693
+ case 'label': return done({ refusal: 'label changes diagnostics' })
1694
+ case 'expect': return done({ refusal: 'expect changes diagnostics' })
1695
+ case 'attempt': return done({ refusal: 'attempt changes commitment' })
1696
+ case 'withCtx': return done({ refusal: 'withCtx is dynamic' })
1697
+ case 'guard': return done({ refusal: 'guard is dynamic' })
1698
+ case 'adjacency': return done({ refusal: 'adjacency depends on parser state' })
1699
+ case 'recover': return done({ refusal: 'recover is effectful' })
1700
+ case 'scanTo': return done({ refusal: 'scanTo is not a lexical run' })
1701
+ case 'sepBy': return done({ refusal: 'separated repetition is not a lexical run' })
1702
+ case 'dispatch': return done({ refusal: 'nested dispatch has semantic outcomes' })
1703
+ case 'routed': return done({ refusal: 'routed depends on an outer dispatch' })
1704
+ case 'unknown': return done({ refusal: 'unknown parser body' })
1705
+ }
1706
+ }
1707
+
1708
+ /**
1709
+ * `balanced()` deliberately overrides its eager structural `_def`; the
1710
+ * constructor marker is the only authoritative recognition language. Model it
1711
+ * directly instead of rediscovering a recursive transform/expect expansion.
1712
+ */
1713
+ function normalizeBalancedLexical(
1714
+ parser: Combinator<unknown>,
1715
+ resolve?: (name: string) => Combinator<unknown> | undefined,
1716
+ session?: LexicalNormalizeSession,
1717
+ ): Normalized | undefined {
1718
+ // `token(balanced(...))` is legal and appears in Jess. The outer token owns
1719
+ // the public leaf, while the inner balanced token owns the constructor marker.
1720
+ let marked = parser
1721
+ let spec = (marked as BalancedSpec)._balancedSpec
1722
+ const tokenWrappers: Combinator<unknown>[] = []
1723
+ while (marked._def.tag === 'token') {
1724
+ tokenWrappers.push(marked)
1725
+ marked = marked._def.parser
1726
+ spec ??= (marked as BalancedSpec)._balancedSpec
1727
+ }
1728
+ if (spec === undefined) return undefined
1729
+ const machine = (): Normalized => withLexicalControl(session, 'balanced', () => {
1730
+ lexicalState(session) // balanced machine
1731
+ const openState = lexicalState(session)
1732
+ if (session !== undefined) lexicalFail(session, openState, directTerminalFailureExpected({
1733
+ tag: 'literal', value: spec.open, caseInsensitive: false,
1734
+ }))
1735
+ lexicalState(session) // balanced body scan
1736
+ const skip: LexicalIr[] = []
1737
+ // Ambient scanSkip is an observable runtime callback policy: balanced keys
1738
+ // its rebuilt interior by the live array identity and dynamically looks up
1739
+ // every callback's `.parse`. Only authored ownSkip belongs to canonical IR.
1740
+ for (const entry of spec.ownSkip) {
1741
+ const normalized = withLexicalControl(session, 'balanced-skip', () =>
1742
+ normalizeBalancedLexical(entry, resolve, session)
1743
+ ?? normalizeLexical(entry, resolve, new Set(), session))
1744
+ if ('refusal' in normalized) return {
1745
+ refusal: `balanced skipper is not represented: ${normalized.refusal}`,
1746
+ }
1747
+ skip.push(normalized.ir)
1748
+ }
1749
+ const closeState = lexicalState(session)
1750
+ if (session !== undefined) {
1751
+ const expectedId = lexicalExpected(session, directTerminalFailureExpected({
1752
+ tag: 'literal', value: spec.close, caseInsensitive: false,
1753
+ }))
1754
+ session.events.push(spec.strict
1755
+ ? {
1756
+ op: 'BAL_CLOSE_STRICT', state: closeState, expectedId, probe: true, committed: false,
1757
+ when: 'close-miss-after-open-body-success',
1758
+ }
1759
+ : {
1760
+ op: 'BAL_CLOSE_RECOVER', state: closeState, expectedId,
1761
+ probe: true, committed: false, when: 'close-miss-after-open-body-success',
1762
+ result: 'parseError', errorSpan: 'close-position',
1763
+ lineAnnotation: 'when-active', errorSink: 'when-active', cstErrorCapture: 'when-active',
1764
+ })
1765
+ }
1766
+ return {
1767
+ ir: {
1768
+ kind: 'balanced', open: spec.open, close: spec.close,
1769
+ strict: spec.strict, raw: spec.raw, skip,
1770
+ },
1771
+ }
1772
+ })
1773
+ const wrap = (index: number): Normalized => {
1774
+ const tokenParser = tokenWrappers[index]
1775
+ return tokenParser === undefined ? machine()
1776
+ : withLexicalControl(session, 'token', () =>
1777
+ withLexicalWrapper(session, 'token', tokenParser, () => wrap(index + 1)))
1778
+ }
1779
+ return wrap(0)
1780
+ }
1781
+
1782
+ const CAPABILITY_VARIANT_GAP =
1783
+ 'total token replacement body is not implemented for every supported assembly variant'
1784
+ const FIXED_TUPLE_BINDING_GAP =
1785
+ 'binding candidate set is incomplete: shared, direct captured, and statically named fixed-tuple bodies have not all been built before pricing; fixed children and routes may not be rediscovered from arrays'
1786
+ const COMPLETE_CAPABILITY = { kind: 'complete' } as const
1787
+ const gap = (reason: string): LexicalCapabilityStatus => ({ kind: 'gap', reason })
1788
+ const COMPLETE_PHASE: LexicalCapabilityPhase = {
1789
+ representation: COMPLETE_CAPABILITY,
1790
+ executableLowering: COMPLETE_CAPABILITY,
1791
+ }
1792
+ const phaseGap = (representation: LexicalCapabilityStatus, loweringReason: string): LexicalCapabilityPhase => ({
1793
+ representation,
1794
+ executableLowering: representation.kind === 'impossible'
1795
+ ? representation : gap(loweringReason),
1796
+ })
1797
+
1798
+ const ALL_BINDING_READERS = 0b111
1799
+ const ALL_BINDING_VARIANTS = 0b11_1111
1800
+ const TERMINAL_CAPTURED_TEMPLATE_ID = 11
1801
+ const DECISION_REFERENCE_TEMPLATE_ID = 12
1802
+ const DECISION_CAPTURED_TEMPLATE_ID = 13
1803
+ const DECISION_NAMED_TEMPLATE_ID = 14
1804
+
1805
+ /** Stable pre-pricing template ids. They name existing reader constructions;
1806
+ * they are deliberately not opcodes and never enter the parse path. */
1807
+ function bindingTemplateId(parentTag: ParserDef['tag'] | 'root', ordinal: number): number {
1808
+ if (parentTag === 'root') return 1
1809
+ switch (parentTag) {
1810
+ case 'sequence': return 2
1811
+ case 'choice': return 3
1812
+ case 'dispatch': return ordinal === 0 ? 4 : 5
1813
+ case 'many': case 'oneOrMore': case 'sepBy': return ordinal === 0 ? 6 : 7
1814
+ case 'scanTo': return ordinal === 0 ? 8 : 9
1815
+ default: return 10
1816
+ }
1817
+ }
1818
+
1819
+ function bindingDigest(words: readonly number[], parentTag: string, childTag: string): number {
1820
+ let hash = 0x811c9dc5
1821
+ const feed = (value: number): void => { hash ^= value & 0xFF; hash = Math.imul(hash, 0x01000193) }
1822
+ for (const c of `${parentTag}\u0000${childTag}`) feed(c.charCodeAt(0))
1823
+ for (const word of words) {
1824
+ feed(word); feed(word >>> 8); feed(word >>> 16); feed(word >>> 24)
1825
+ }
1826
+ return hash >>> 0
1827
+ }
1828
+
1829
+ function fixedBindingProjection(
1830
+ edge: BindingEdgeCandidate,
1831
+ edgeId: number,
1832
+ projectionId: number,
1833
+ ): LexicalBindingProjection {
1834
+ const template = bindingTemplateId(edge.parentTag, edge.childOrdinal)
1835
+ let readerMask = ALL_BINDING_READERS
1836
+ const variantMask = ALL_BINDING_VARIANTS
1837
+ // Scanner children already have one fixed projection per reader: execRules
1838
+ // uses their numeric SubtreeRef, the closure adapter captures `pieceAt(ip)`,
1839
+ // and the emitted adapter captures that site's named body after linking every
1840
+ // scan root. Only ambient trivia/scanSkip remain live runtime callbacks; they
1841
+ // are scanner POLICY, never binding authority for these authored own children.
1842
+ // Recovery/withCtx still have no equivalent named template.
1843
+ if (edge.parentTag === 'recover' || edge.parentTag === 'withCtx') readerMask = 0b011
1844
+ // Sequence strict, adjacency, and recovery bodies now all use scalar block
1845
+ // chains, so no supported variant is withheld here.
1846
+ const words = [
1847
+ edge.parentBodyId, edge.childBodyId, edge.childOrdinal,
1848
+ edge.childOrdinal, edge.childOrdinal, template,
1849
+ edge.childBodyId, template, readerMask, variantMask,
1850
+ ]
1851
+ return {
1852
+ id: projectionId, edgeId,
1853
+ parentBodyId: edge.parentBodyId, childBodyId: edge.childBodyId,
1854
+ childOrdinal: edge.childOrdinal, referenceOperand: edge.childOrdinal,
1855
+ capturedSlotId: edge.childOrdinal, capturedTemplateId: template,
1856
+ namedSymbolId: edge.childBodyId, namedTemplateId: template,
1857
+ readerMask, variantMask,
1858
+ semanticDigest: bindingDigest(words, edge.parentTag, edge.childTag),
1859
+ }
1860
+ }
1861
+
1862
+ function derivedCapabilityStatus(obligations: LexicalCapabilityObligations): LexicalCapabilityStatus {
1863
+ const phases = [
1864
+ obligations.recognition,
1865
+ obligations.diagnostics,
1866
+ obligations.boundaryPlan,
1867
+ obligations.materializationPlan,
1868
+ obligations.supportedVariants,
1869
+ obligations.bindingAndReachability,
1870
+ ]
1871
+ const ordered = phases.flatMap(entry => [entry.representation, entry.executableLowering])
1872
+ const impossible = ordered.find((entry): entry is Extract<LexicalCapabilityStatus, { kind: 'impossible' }> =>
1873
+ entry.kind === 'impossible')
1874
+ if (impossible !== undefined) return impossible
1875
+ // Capability is availability before selection: only executable lowering can
1876
+ // close it. Actual execution is test evidence, not a third status dimension.
1877
+ const gaps = phases.map(entry => entry.executableLowering)
1878
+ .filter((entry): entry is Extract<LexicalCapabilityStatus, { kind: 'gap' }> =>
1879
+ entry.kind === 'gap')
1880
+ if (gaps.length > 0) return gap([...new Set(gaps.map(entry => entry.reason))].join('; '))
1881
+ return COMPLETE_CAPABILITY
1882
+ }
1883
+
1884
+ function tokenObligations(
1885
+ recognition: LexicalCapabilityStatus,
1886
+ represented: {
1887
+ readonly diagnostics: boolean
1888
+ readonly boundaryPlan: LexicalCapabilityStatus
1889
+ readonly materializationPlan: LexicalCapabilityStatus
1890
+ } = {
1891
+ diagnostics: false,
1892
+ boundaryPlan: gap('token boundary transaction is not represented'),
1893
+ materializationPlan: gap('token range/value/CST materialization is not represented'),
1894
+ },
1895
+ ): LexicalCapabilityObligations {
1896
+ const representedOrGap = (yes: boolean, reason: string): LexicalCapabilityStatus =>
1897
+ yes ? COMPLETE_CAPABILITY : gap(reason)
1898
+ return {
1899
+ recognition: phaseGap(recognition, 'token recognizer lowering is not implemented for every supported variant'),
1900
+ diagnostics: phaseGap(
1901
+ representedOrGap(represented.diagnostics, 'token diagnostic plan is not represented'),
1902
+ 'token diagnostic event lowering is deferred'),
1903
+ boundaryPlan: phaseGap(
1904
+ represented.boundaryPlan,
1905
+ 'token boundary transaction lowering is deferred'),
1906
+ materializationPlan: phaseGap(
1907
+ represented.materializationPlan,
1908
+ 'token range/value/CST materialization lowering is deferred'),
1909
+ supportedVariants: phaseGap(gap(CAPABILITY_VARIANT_GAP), CAPABILITY_VARIANT_GAP),
1910
+ bindingAndReachability: phaseGap(gap(FIXED_TUPLE_BINDING_GAP), FIXED_TUPLE_BINDING_GAP),
1911
+ }
1912
+ }
1913
+
1914
+ function executableLexBodyObligations(): LexicalCapabilityObligations {
1915
+ return {
1916
+ recognition: COMPLETE_PHASE,
1917
+ diagnostics: COMPLETE_PHASE,
1918
+ boundaryPlan: COMPLETE_PHASE,
1919
+ materializationPlan: COMPLETE_PHASE,
1920
+ supportedVariants: COMPLETE_PHASE,
1921
+ bindingAndReachability: COMPLETE_PHASE,
1922
+ }
1923
+ }
1924
+
1925
+ function terminalObligations(): LexicalCapabilityObligations {
1926
+ return {
1927
+ recognition: COMPLETE_PHASE,
1928
+ diagnostics: COMPLETE_PHASE,
1929
+ boundaryPlan: COMPLETE_PHASE,
1930
+ materializationPlan: COMPLETE_PHASE,
1931
+ supportedVariants: COMPLETE_PHASE,
1932
+ bindingAndReachability: phaseGap(gap(FIXED_TUPLE_BINDING_GAP), FIXED_TUPLE_BINDING_GAP),
1933
+ }
1934
+ }
1935
+
1936
+ function decisionObligations(recognition: LexicalCapabilityStatus): LexicalCapabilityObligations {
1937
+ return tokenObligations(recognition)
1938
+ }
1939
+
1940
+ /** A final decision does not own a token boundary or token materialization.
1941
+ * Its executable projection is the existing canonical TableProgram row plus
1942
+ * the already-validated fixed child/effect projections. */
1943
+ function executableDecisionObligations(
1944
+ recognition: LexicalCapabilityStatus,
1945
+ effects: LexicalCapabilityPhase,
1946
+ ): LexicalCapabilityObligations {
1947
+ const recognitionPhase = recognition.kind === 'complete'
1948
+ ? COMPLETE_PHASE
1949
+ : phaseGap(recognition, 'decision recognition authority is incomplete')
1950
+ return {
1951
+ recognition: recognitionPhase,
1952
+ diagnostics: effects,
1953
+ boundaryPlan: COMPLETE_PHASE,
1954
+ materializationPlan: COMPLETE_PHASE,
1955
+ supportedVariants: effects,
1956
+ bindingAndReachability: effects,
1957
+ }
1958
+ }
1959
+
1960
+ type DecisionLead = {
1961
+ readonly parser: Combinator<unknown>
1962
+ readonly ir: LexicalIr
1963
+ readonly key: string
1964
+ readonly dispatch?: Extract<ParserDef, { tag: 'dispatch' }>
1965
+ }
1966
+
1967
+ type DecisionLeadResult = { readonly lead: DecisionLead } | { readonly gap: string }
1968
+
1969
+ /**
1970
+ * Find the one atomic range whose recognition starts this arm. This is only a
1971
+ * language proof: wrappers still owe diagnostics/effects and remain in the
1972
+ * eventual replacement body. Nullable prefixes and context-changing wrappers
1973
+ * deliberately decline the pruning proof rather than being guessed through.
1974
+ * The authored arm remains an explicit unrestricted TOKEN-body candidate; a
1975
+ * declined relation is not a missing recognizer and never licenses replay.
1976
+ */
1977
+ function decisionLead(
1978
+ parser: Combinator<unknown>,
1979
+ context: CapabilityContext,
1980
+ resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
1981
+ seen = new Set<Combinator<unknown>>(),
1982
+ ): DecisionLeadResult {
1983
+ if (seen.has(parser)) return { gap: 'recursive leading decision language' }
1984
+ seen.add(parser)
1985
+ const finish = (result: DecisionLeadResult): DecisionLeadResult => { seen.delete(parser); return result }
1986
+ const def = parser._def
1987
+ if (keyOf(def) !== undefined || def.tag === 'token' || (parser as BalancedSpec)._balancedSpec !== undefined) {
1988
+ const normalized = normalizeBalancedLexical(parser, resolve)
1989
+ ?? normalizeLexical(def.tag === 'token' ? def.parser : parser, resolve, new Set())
1990
+ if ('refusal' in normalized) return finish({ gap: `leading recognizer: ${normalized.refusal}` })
1991
+ return finish({ lead: {
1992
+ parser,
1993
+ ir: normalized.ir,
1994
+ key: JSON.stringify(normalized.ir),
1995
+ } })
1996
+ }
1997
+ switch (def.tag) {
1998
+ case 'dispatch': {
1999
+ const selected = decisionLead(def.selector, context, resolve, seen)
2000
+ if ('gap' in selected) return finish(selected)
2001
+ return finish({ lead: { ...selected.lead, dispatch: def } })
2002
+ }
2003
+ case 'lazy': {
2004
+ const target = resolvedLazyTarget(parser, resolve)
2005
+ return finish(target === undefined
2006
+ ? { gap: 'unresolved leading rule reference' }
2007
+ : decisionLead(target, context, resolve, seen))
2008
+ }
2009
+ case 'sequence':
2010
+ if (def.parsers.length === 0) return finish({ gap: 'empty leading sequence' })
2011
+ if (matchesEmpty(def.parsers[0]!, new Set(), resolve)) {
2012
+ return finish({ gap: 'nullable leading sequence needs a total same-position union proof' })
2013
+ }
2014
+ return finish(decisionLead(def.parsers[0]!, context, resolve, seen))
2015
+ case 'attempt': case 'transform': case 'leaf': case 'node': case 'field': case 'label': case 'expect':
2016
+ case 'trivia':
2017
+ return finish(decisionLead(def.parser, context, resolve, seen))
2018
+ case 'grammar':
2019
+ if (def.triviaParser !== undefined || def.clearTrivia === true || def.trackLines
2020
+ || def.captureTrivia === true || def.rootCapture !== undefined) {
2021
+ return finish({ gap: 'leading grammar wrapper changes lexical context' })
2022
+ }
2023
+ return finish(decisionLead(def.parser, context, resolve, seen))
2024
+ case 'choice': return finish({ gap: 'nested leading choice needs a compatible range union proof' })
2025
+ case 'optional': case 'many': case 'oneOrMore': case 'sepBy':
2026
+ return finish({ gap: 'repeated or nullable leading language needs a range-partition proof' })
2027
+ case 'peek': return finish({ gap: 'positive lookahead needs an exact effect-free compatible view' })
2028
+ case 'not': return finish({ gap: 'negative lookahead needs a decidable complement' })
2029
+ case 'withCtx': case 'guard': case 'adjacency': case 'recover': case 'scanTo': case 'routed': case 'unknown':
2030
+ return finish({ gap: `leading ${def.tag} language is not represented` })
2031
+ case 'literal': case 'keywords': case 'regex':
2032
+ throw new Error('parseman: decision lead terminal escaped its direct case')
2033
+ }
2034
+ }
2035
+
2036
+ function finiteLexicalValues(ir: LexicalIr): readonly string[] | undefined {
2037
+ if (ir.kind === 'literal') return [ir.value]
2038
+ if (ir.kind === 'keywords') return ir.words
2039
+ return undefined
2040
+ }
2041
+
2042
+ function lexicalIrEnd(ir: LexicalIr, input: string, start = 0): number | undefined {
2043
+ switch (ir.kind) {
2044
+ case 'literal': {
2045
+ const actual = input.slice(start, start + ir.value.length)
2046
+ return (ir.caseInsensitive ? foldAscii(actual) === foldAscii(ir.value) : actual === ir.value)
2047
+ ? start + ir.value.length : undefined
2048
+ }
2049
+ case 'keywords': {
2050
+ for (const word of ir.words) {
2051
+ const actual = input.slice(start, start + word.length)
2052
+ if (ir.caseInsensitive ? foldAscii(actual) === foldAscii(word) : actual === word) {
2053
+ const next = input[start + word.length]
2054
+ if (next === undefined || ir.boundary === undefined) return start + word.length
2055
+ try {
2056
+ if (!new RegExp(`[${ir.boundary}]`).test(next)) return start + word.length
2057
+ } catch { return undefined }
2058
+ }
2059
+ }
2060
+ return undefined
2061
+ }
2062
+ case 'regex': {
2063
+ try {
2064
+ // regex() performs one authored sticky match. Anchoring the source in a
2065
+ // wrapper changes both alternation backtracking (`a|ab`) and multiline
2066
+ // `$` semantics, so proof must inspect the actual chosen end.
2067
+ const matcher = new RegExp(ir.source, `${regexFlags(ir.flags)}y`)
2068
+ matcher.lastIndex = start
2069
+ const match = matcher.exec(input)
2070
+ return match === null ? undefined : matcher.lastIndex
2071
+ } catch { return undefined }
2072
+ }
2073
+ case 'choice': {
2074
+ for (const arm of ir.arms) {
2075
+ const end = lexicalIrEnd(arm, input, start)
2076
+ if (end !== undefined) return end
2077
+ }
2078
+ return undefined
2079
+ }
2080
+ case 'sequence': {
2081
+ let end = start
2082
+ for (const part of ir.parts) {
2083
+ const next = lexicalIrEnd(part, input, end)
2084
+ if (next === undefined) return undefined
2085
+ end = next
2086
+ }
2087
+ return end
2088
+ }
2089
+ case 'repeat': {
2090
+ let end = start
2091
+ let count = 0
2092
+ while (ir.max === null || count < ir.max) {
2093
+ const next = lexicalIrEnd(ir.body, input, end)
2094
+ if (next === undefined || next === end) break
2095
+ end = next
2096
+ count++
2097
+ }
2098
+ return count >= ir.min ? end : undefined
2099
+ }
2100
+ case 'assert': {
2101
+ const matched = lexicalIrEnd(ir.body, input, start) !== undefined
2102
+ return matched === ir.positive ? start : undefined
2103
+ }
2104
+ case 'balanced': return undefined
2105
+ }
2106
+ }
2107
+
2108
+ function lexicalIrAcceptsExact(ir: LexicalIr, value: string): boolean {
2109
+ return lexicalIrEnd(ir, value) === value.length
2110
+ }
2111
+
2112
+ function asciiCaseVariants(value: string, caseInsensitive: boolean): readonly string[] | undefined {
2113
+ if (caseInsensitive && /[^\x00-\x7f]/.test(value)) return undefined
2114
+ if (!caseInsensitive || !/[A-Za-z]/.test(value)) return [value]
2115
+ let variants = ['']
2116
+ for (const char of value) {
2117
+ const lower = char >= 'A' && char <= 'Z' ? char.toLowerCase() : char
2118
+ const upper = char >= 'a' && char <= 'z' ? char.toUpperCase() : char
2119
+ if (lower !== upper && variants.length > 2048) return undefined
2120
+ variants = lower === upper
2121
+ ? variants.map(prefix => prefix + char)
2122
+ : variants.flatMap(prefix => [prefix + lower, prefix + upper])
2123
+ }
2124
+ return variants
2125
+ }
2126
+
2127
+ function regexAsciiCaseStable(source: string): boolean {
2128
+ let index = 0
2129
+ while (index < source.length) {
2130
+ const char = source[index]!
2131
+ if (char === '\\') {
2132
+ const kind = source[index + 1]
2133
+ if (kind === 'x') {
2134
+ const value = Number.parseInt(source.slice(index + 2, index + 4), 16)
2135
+ if (Number.isFinite(value) && /[A-Za-z]/.test(String.fromCharCode(value))) return false
2136
+ index += 4
2137
+ continue
2138
+ }
2139
+ if (kind === 'u') {
2140
+ const brace = source[index + 2] === '{'
2141
+ const end = brace ? source.indexOf('}', index + 3) : index + 6
2142
+ const digits = brace ? source.slice(index + 3, end) : source.slice(index + 2, end)
2143
+ const value = Number.parseInt(digits, 16)
2144
+ if (Number.isFinite(value) && value <= 0xffff
2145
+ && /[A-Za-z]/.test(String.fromCharCode(value))) return false
2146
+ index = brace ? (end + 1 || source.length) : end
2147
+ continue
2148
+ }
2149
+ if (kind !== undefined && 'dDsSwWpP'.includes(kind)) { index += 2; continue }
2150
+ if (kind !== undefined && /[A-Za-z]/.test(kind)) return false
2151
+ index += 2
2152
+ continue
2153
+ }
2154
+ if (char === '[') {
2155
+ let end = index + 1
2156
+ for (; end < source.length; end++) {
2157
+ if (source[end] === '\\') { end++; continue }
2158
+ if (source[end] === ']') break
2159
+ }
2160
+ if (end >= source.length) return false
2161
+ const body = source.slice(index + 1, end)
2162
+ const lowerRange = body.includes('a-z')
2163
+ const upperRange = body.includes('A-Z')
2164
+ if (lowerRange !== upperRange) return false
2165
+ const withoutRanges = body.replace(/a-z|A-Z/g, '')
2166
+ const literals = new Set<string>()
2167
+ for (let at = 0; at < withoutRanges.length; at++) {
2168
+ if (withoutRanges[at] === '\\') {
2169
+ const kind = withoutRanges[++at]
2170
+ if (kind === 'x') {
2171
+ const value = Number.parseInt(withoutRanges.slice(at + 1, at + 3), 16)
2172
+ if (Number.isFinite(value) && /[A-Za-z]/.test(String.fromCharCode(value))) {
2173
+ literals.add(String.fromCharCode(value))
2174
+ }
2175
+ at += 2
2176
+ }
2177
+ else if (kind === 'u') {
2178
+ let digits: string
2179
+ if (withoutRanges[at + 1] === '{') {
2180
+ const end = withoutRanges.indexOf('}', at + 2)
2181
+ if (end < 0) return false
2182
+ digits = withoutRanges.slice(at + 2, end)
2183
+ at = end
2184
+ } else {
2185
+ digits = withoutRanges.slice(at + 1, at + 5)
2186
+ at += 4
2187
+ }
2188
+ const value = Number.parseInt(digits, 16)
2189
+ if (Number.isFinite(value) && value <= 0xffff
2190
+ && /[A-Za-z]/.test(String.fromCharCode(value))) {
2191
+ literals.add(String.fromCharCode(value))
2192
+ }
2193
+ } else if (kind !== undefined && /[A-Za-z]/.test(kind)
2194
+ && !'dDsSwWpP'.includes(kind)) return false
2195
+ continue
2196
+ }
2197
+ if (/[A-Za-z]/.test(withoutRanges[at]!)) literals.add(withoutRanges[at]!)
2198
+ }
2199
+ for (const letter of literals) {
2200
+ const peer = letter === letter.toLowerCase() ? letter.toUpperCase() : letter.toLowerCase()
2201
+ if (!literals.has(peer)) return false
2202
+ }
2203
+ index = end + 1
2204
+ continue
2205
+ }
2206
+ if (/[A-Za-z]/.test(char)) return false
2207
+ index++
2208
+ }
2209
+ return true
2210
+ }
2211
+
2212
+ function lexicalIrAsciiCaseStable(ir: LexicalIr): boolean {
2213
+ switch (ir.kind) {
2214
+ case 'literal': return ir.caseInsensitive || !/[A-Za-z]/.test(ir.value)
2215
+ case 'keywords': return ir.caseInsensitive
2216
+ || ir.words.every(word => !/[A-Za-z]/.test(word))
2217
+ case 'regex': return ir.flags.includes('i') || regexAsciiCaseStable(ir.source)
2218
+ case 'sequence': return ir.parts.every(lexicalIrAsciiCaseStable)
2219
+ case 'choice': return ir.arms.every(lexicalIrAsciiCaseStable)
2220
+ case 'repeat': case 'assert': return lexicalIrAsciiCaseStable(ir.body)
2221
+ case 'balanced': return !/[A-Za-z]/.test(ir.open + ir.close)
2222
+ && ir.skip.every(lexicalIrAsciiCaseStable)
2223
+ }
2224
+ }
2225
+
2226
+ type ContinuationClass = { readonly source: string; readonly flags: string }
2227
+
2228
+ function lexicalContinuationClasses(ir: LexicalIr): readonly ContinuationClass[] | undefined {
2229
+ if (ir.kind === 'regex') {
2230
+ const classesRemoved = ir.source.replace(/\[(?:\\.|[^\]\\])*\]/g, '[]')
2231
+ if (/[|()$^]/.test(classesRemoved) || /\\[bB]/.test(classesRemoved)) return undefined
2232
+ const tail = /(\[(?:\\.|[^\]\\])+\])(?:\*|\+)$/.exec(ir.source)
2233
+ if (tail === null || /[uv]/.test(ir.flags) || /\\[pP]|\\u\{/.test(tail[1]!)) return undefined
2234
+ return [{ source: tail[1]!, flags: ir.flags }]
2235
+ }
2236
+ if (ir.kind === 'sequence' && ir.parts.length === 2) {
2237
+ const [head, tail] = ir.parts
2238
+ if (tail?.kind !== 'repeat' || tail.min !== 0 || tail.max !== 1
2239
+ || tail.body.kind !== 'literal' || [...tail.body.value].length !== 1) return undefined
2240
+ const headClasses = lexicalContinuationClasses(head!)
2241
+ if (headClasses === undefined) return undefined
2242
+ const escaped = tail.body.value.replace(/[\\\]\-^]/g, '\\$&')
2243
+ return [...headClasses, {
2244
+ source: `[${escaped}]`, flags: tail.body.caseInsensitive ? 'i' : '',
2245
+ }]
2246
+ }
2247
+ return undefined
2248
+ }
2249
+
2250
+ function boundaryCoversContinuation(family: LexicalIr, boundary: string): boolean {
2251
+ const classes = lexicalContinuationClasses(family)
2252
+ if (classes === undefined) return false
2253
+ let boundaryMatcher: RegExp
2254
+ const matchers: RegExp[] = []
2255
+ try {
2256
+ boundaryMatcher = new RegExp(`^[${boundary}]$`)
2257
+ for (const entry of classes) {
2258
+ matchers.push(new RegExp(`^${entry.source}$`, regexFlags(entry.flags)))
2259
+ }
2260
+ } catch { return false }
2261
+ for (let code = 0; code <= 0xffff; code++) {
2262
+ const char = String.fromCharCode(code)
2263
+ if (matchers.some(matcher => matcher.test(char)) && !boundaryMatcher.test(char)) return false
2264
+ }
2265
+ return true
2266
+ }
2267
+
2268
+ function fixedRegexLiteral(source: string): string | undefined {
2269
+ let value = ''
2270
+ for (let index = 0; index < source.length; index++) {
2271
+ const char = source[index]!
2272
+ if (char !== '\\') {
2273
+ if (/[.[\]{}()*+?|^$]/.test(char)) return undefined
2274
+ value += char
2275
+ continue
2276
+ }
2277
+ const kind = source[++index]
2278
+ if (kind === undefined || 'dDsSwWpPbB'.includes(kind)) return undefined
2279
+ if (kind === 'x') {
2280
+ const code = Number.parseInt(source.slice(index + 1, index + 3), 16)
2281
+ if (!Number.isFinite(code)) return undefined
2282
+ value += String.fromCharCode(code)
2283
+ index += 2
2284
+ continue
2285
+ }
2286
+ if (kind === 'u') {
2287
+ if (source[index + 1] === '{') return undefined
2288
+ const code = Number.parseInt(source.slice(index + 1, index + 5), 16)
2289
+ if (!Number.isFinite(code)) return undefined
2290
+ value += String.fromCharCode(code)
2291
+ index += 4
2292
+ continue
2293
+ }
2294
+ value += kind
2295
+ }
2296
+ return value
2297
+ }
2298
+
2299
+ function familyHasContextFreeFiniteEnd(
2300
+ ir: LexicalIr,
2301
+ spellings: readonly string[],
2302
+ ): boolean {
2303
+ if (ir.kind === 'literal') return true
2304
+ if (ir.kind === 'regex') return fixedRegexLiteral(ir.source) !== undefined
2305
+ if (ir.kind !== 'keywords' || ir.boundary !== undefined) return false
2306
+ return spellings.every(spelling => !ir.words.some(word => {
2307
+ const actual = ir.caseInsensitive ? foldAscii(spelling) : spelling
2308
+ const candidate = ir.caseInsensitive ? foldAscii(word) : word
2309
+ return candidate.length > actual.length && candidate.startsWith(actual)
2310
+ }))
2311
+ }
2312
+
2313
+ function compatibleLanguageViews(
2314
+ arm: LexicalIr,
2315
+ family: LexicalIr,
2316
+ ): ReadonlyArray<{ readonly ir: LexicalIr; readonly relation: 'equal' | 'prefix' }> | undefined {
2317
+ if (JSON.stringify(arm) === JSON.stringify(family)) return [{ ir: arm, relation: 'equal' }]
2318
+ const values = finiteLexicalValues(arm)
2319
+ if (values === undefined || values.length === 0) return undefined
2320
+ const familyValues = finiteLexicalValues(family)
2321
+ const foldedArm = arm.kind === 'literal' ? arm.caseInsensitive : arm.kind === 'keywords' && arm.caseInsensitive
2322
+ const foldedFamily = family.kind === 'literal'
2323
+ ? family.caseInsensitive : family.kind === 'keywords' && family.caseInsensitive
2324
+ const armBoundary = arm.kind === 'keywords' ? arm.boundary : undefined
2325
+ const views: Array<{ ir: LexicalIr; relation: 'equal' | 'prefix' }> = []
2326
+ for (const value of values) {
2327
+ let relation: 'equal' | 'prefix' | undefined
2328
+ const armCaseInsensitive = arm.kind === 'literal'
2329
+ ? arm.caseInsensitive : arm.kind === 'keywords' && arm.caseInsensitive
2330
+ const spellings = armCaseInsensitive && !/[^\x00-\x7f]/.test(value)
2331
+ && lexicalIrAsciiCaseStable(family)
2332
+ ? [value] : asciiCaseVariants(value, armCaseInsensitive)
2333
+ const endIsTotal = spellings !== undefined
2334
+ && (familyHasContextFreeFiniteEnd(family, spellings)
2335
+ || armBoundary !== undefined && boundaryCoversContinuation(family, armBoundary))
2336
+ if (endIsTotal && spellings !== undefined
2337
+ && spellings.every(spelling => lexicalIrAcceptsExact(family, spelling))) {
2338
+ relation = 'equal'
2339
+ }
2340
+ else if (familyValues !== undefined
2341
+ && (family.kind === 'literal' || family.kind === 'keywords' && family.boundary === undefined)
2342
+ && foldedArm === foldedFamily && familyValues.some(full => {
2343
+ const a = foldedArm ? foldAscii(value) : value
2344
+ const f = foldedFamily ? foldAscii(full) : full
2345
+ if (!f.startsWith(a) || f.length === a.length) return false
2346
+ if (arm.kind !== 'keywords' || arm.boundary === undefined) return true
2347
+ try { return !new RegExp(`[${arm.boundary}]`).test(full[a.length]!) } catch { return false }
2348
+ })) relation = 'prefix'
2349
+ if (relation === undefined) return undefined
2350
+ const ir: LexicalIr = arm.kind === 'keywords'
2351
+ ? { ...arm, words: [value] }
2352
+ : arm.kind === 'literal'
2353
+ ? { ...arm, value }
2354
+ : (() => { throw new Error('parseman: finite decision view lost its language') })()
2355
+ views.push({ ir, relation })
2356
+ }
2357
+ return views
2358
+ }
2359
+
2360
+ type PendingDecisionView = {
2361
+ readonly familyKey: string
2362
+ readonly key: string
2363
+ readonly view: LexicalDecisionOutcomeView
2364
+ }
2365
+
2366
+ type PendingAcceptance =
2367
+ | { readonly kind: 'outcomes'; readonly keys: readonly string[] }
2368
+ | { readonly kind: 'otherwise'; readonly excludingKeys: readonly string[] }
2369
+ | Exclude<LexicalDecisionAcceptance, { kind: 'outcomes' | 'otherwise' }>
2370
+
2371
+ type PendingDecisionSite = {
2372
+ readonly siteId: number
2373
+ readonly atom: 'choice' | 'dispatch'
2374
+ readonly path: string
2375
+ readonly contextKey: string
2376
+ readonly contextSnapshotId: number
2377
+ readonly families: ReadonlyArray<{
2378
+ readonly familyKey: string
2379
+ readonly arms: ReadonlyArray<Omit<LexicalDecisionArm, 'acceptance'> & { readonly acceptance: PendingAcceptance }>
2380
+ }>
2381
+ readonly precisionNotes: readonly string[]
2382
+ readonly pendingReuse: 'forbidden' | 'unproved'
2383
+ readonly reuseEpochs: readonly LexicalDecisionReuseEpoch[]
2384
+ readonly routeUsesRouted?: readonly boolean[]
2385
+ }
2386
+
2387
+ type ScanConsumerCandidate = {
2388
+ readonly path: string
2389
+ readonly contextKey: string
2390
+ readonly context: CapabilityContext
2391
+ readonly kind: 'scanTo' | 'balanced'
2392
+ readonly raw: boolean
2393
+ readonly ownSkip: readonly Combinator<unknown>[]
2394
+ }
2395
+
2396
+ function balancedSpecOf(parser: Combinator<unknown>): BalancedSpec['_balancedSpec'] {
2397
+ let current = parser
2398
+ let spec = (current as BalancedSpec)._balancedSpec
2399
+ const seen = new Set<Combinator<unknown>>()
2400
+ while (spec === undefined && current._def.tag === 'token' && !seen.has(current)) {
2401
+ seen.add(current)
2402
+ current = current._def.parser
2403
+ spec = (current as BalancedSpec)._balancedSpec
2404
+ }
2405
+ return spec
2406
+ }
2407
+
2408
+ function ownedScanConsumers(
2409
+ root: Combinator<unknown>,
2410
+ initialContext: CapabilityContext,
2411
+ contextKeyOf: (context: CapabilityContext) => string,
2412
+ isSeparateOccurrence: (parser: Combinator<unknown>, contextKey: string) => boolean,
2413
+ resolve?: (name: string) => Combinator<unknown> | undefined,
2414
+ ): ScanConsumerCandidate[] {
2415
+ const out: ScanConsumerCandidate[] = []
2416
+ const stack = new Set<Combinator<unknown>>()
2417
+ const walk = (
2418
+ parser: Combinator<unknown>, path: string, first: boolean, context: CapabilityContext,
2419
+ ): void => {
2420
+ if (stack.has(parser)) return
2421
+ const def = parser._def
2422
+ const contextKey = contextKeyOf(context)
2423
+ // Stop only at an independently inventoried occurrence. Nested atomic
2424
+ // wrappers inside token() are recognition machinery owned by the outer site.
2425
+ if (!first && isSeparateOccurrence(parser, contextKey)) return
2426
+ let childContext = context
2427
+ if (def.tag === 'grammar') childContext = {
2428
+ trivia: def.clearTrivia ? undefined : (def.triviaParser ?? context.trivia),
2429
+ scanSkip: context.scanSkip,
2430
+ trackLines: context.trackLines || def.trackLines,
2431
+ captureTrivia: context.captureTrivia || def.captureTrivia === true,
2432
+ opaqueRootCapture: context.opaqueRootCapture || def.rootCapture === 'opaque',
2433
+ dynamicState: context.dynamicState,
2434
+ }
2435
+ else if (def.tag === 'withCtx') childContext = { ...context, dynamicState: true }
2436
+ // token() clears trivia and collector buffers, but the public
2437
+ // ctx.captureTrivia property remains observable by authored callbacks.
2438
+ else if (def.tag === 'token') childContext = { ...context, trivia: undefined }
2439
+ else if (def.tag === 'trivia') childContext = { ...context, trivia: undefined }
2440
+ const balanced = balancedSpecOf(parser)
2441
+ if (balanced !== undefined) {
2442
+ out.push({
2443
+ path, contextKey: contextKeyOf(childContext), context: childContext,
2444
+ kind: 'balanced', raw: balanced.raw, ownSkip: balanced.ownSkip,
2445
+ })
2446
+ return
2447
+ }
2448
+ if (def.tag === 'scanTo') {
2449
+ // scanTo(raw) skips nothing, including its authored `skip` option.
2450
+ out.push({
2451
+ path, contextKey, context, kind: 'scanTo', raw: def.raw, ownSkip: def.raw ? [] : def.skip,
2452
+ })
2453
+ return
2454
+ }
2455
+ stack.add(parser)
2456
+ const children = tokenChildren(parser, resolve)
2457
+ for (let i = 0; i < children.length; i++) {
2458
+ walk(children[i]!, `${path}/${i}`, false, childContext)
2459
+ }
2460
+ stack.delete(parser)
2461
+ }
2462
+ walk(root, '', true, initialContext)
2463
+ return out
2464
+ }
2465
+
2466
+ function containsDynamicScanConsumer(
2467
+ root: Combinator<unknown>,
2468
+ resolve?: (name: string) => Combinator<unknown> | undefined,
2469
+ seen = new Set<Combinator<unknown>>(),
2470
+ ): boolean {
2471
+ if (seen.has(root)) return false
2472
+ if (balancedSpecOf(root) !== undefined || root._def.tag === 'scanTo') return true
2473
+ seen.add(root)
2474
+ for (const child of tokenChildren(root, resolve)) {
2475
+ if (containsDynamicScanConsumer(child, resolve, seen)) return true
2476
+ }
2477
+ return false
2478
+ }
2479
+
2480
+ function predicateViews(def: Extract<ParserDef, { tag: 'dispatch' }>): Array<{
2481
+ readonly match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>
2482
+ readonly armId: number
2483
+ readonly parser: Combinator<unknown>
2484
+ readonly usesRouted: boolean
2485
+ }> {
2486
+ const out: Array<{
2487
+ match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>
2488
+ armId: number
2489
+ parser: Combinator<unknown>
2490
+ usesRouted: boolean
2491
+ }> = []
2492
+ let armId = 0
2493
+ for (const entry of def.cases) {
2494
+ for (const value of entry.keys) out.push({
2495
+ match: { kind: 'exact', values: [value], caseInsensitive: entry.caseInsensitive },
2496
+ armId, parser: entry.parser, usesRouted: branchUsesRouted(entry),
2497
+ })
2498
+ armId++
2499
+ }
2500
+ for (const entry of def.matchers ?? []) {
2501
+ out.push({
2502
+ match: entry.kind === 'matches'
2503
+ ? { kind: 'matches', value: entry.value, flags: entry.flags ?? '', caseInsensitive: entry.caseInsensitive }
2504
+ : { kind: entry.kind, value: entry.value, caseInsensitive: entry.caseInsensitive },
2505
+ armId, parser: entry.parser, usesRouted: branchUsesRouted(entry),
2506
+ })
2507
+ armId++
2508
+ }
2509
+ return out
2510
+ }
2511
+
2512
+ function predicatePartitionStableFor(
2513
+ ir: LexicalIr,
2514
+ predicates: readonly Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[],
2515
+ ): boolean {
2516
+ const folded = ir.kind === 'literal' ? ir.caseInsensitive : ir.kind === 'keywords' && ir.caseInsensitive
2517
+ const values = finiteLexicalValues(ir) ?? []
2518
+ if (!folded) return true
2519
+ if (values.some(value => /[^\x00-\x7f]/.test(value))) return false
2520
+ if (!values.some(value => /[A-Za-z]/.test(value))) return true
2521
+ return predicates.every(match => {
2522
+ if (match.kind === 'matches') return match.caseInsensitive || match.flags.includes('i')
2523
+ if (match.kind === 'exact') return match.caseInsensitive
2524
+ || match.values.every(value => !/[A-Za-z]/.test(value))
2525
+ return match.caseInsensitive || !/[A-Za-z]/.test(match.value)
2526
+ })
2527
+ }
2528
+
2529
+ type CapabilityCandidate = {
2530
+ readonly bodyId: number
2531
+ readonly parser: Combinator<unknown>
2532
+ readonly atom: 'terminal' | 'token' | 'choice' | 'dispatch'
2533
+ readonly path: string
2534
+ readonly context: CapabilityContext
2535
+ readonly contextKey: string
2536
+ readonly recognitionContextKey: string
2537
+ }
2538
+
2539
+ type BindingEdgeCandidate = Omit<
2540
+ LexicalBindingEdge,
2541
+ 'id' | 'status' | 'contextSnapshotId' | 'projectionId'
2542
+ > & {
2543
+ readonly parentBodyId: number
2544
+ readonly childBodyId: number
2545
+ readonly childOrdinal: number
2546
+ }
2547
+
2548
+ type CapabilityContext = LexicalCapabilityContext
2549
+
2550
+ /**
2551
+ * Enumerate atomic lexical ownership boundaries over the final graph. A token
2552
+ * owns its interior, so its private terminals are not separate obligations;
2553
+ * the same terminal reached independently outside that token remains one.
2554
+ */
2555
+ function lexicalCapabilityCandidates(
2556
+ roots: ReadonlyArray<Combinator<unknown>>,
2557
+ resolve?: (name: string) => Combinator<unknown> | undefined,
2558
+ ): {
2559
+ candidates: CapabilityCandidate[]
2560
+ bindingEdges: BindingEdgeCandidate[]
2561
+ contexts: Map<string, CapabilityContext>
2562
+ contextKeyOf: (context: CapabilityContext) => string
2563
+ } {
2564
+ const parserIds = new Map<Combinator<unknown>, number>()
2565
+ const parserId = (parser: Combinator<unknown>): number => {
2566
+ const prior = parserIds.get(parser)
2567
+ if (prior !== undefined) return prior
2568
+ const id = parserIds.size
2569
+ parserIds.set(parser, id)
2570
+ return id
2571
+ }
2572
+ const contextKeyOf = (context: CapabilityContext): string =>
2573
+ `t${context.trivia === undefined ? -1 : parserId(context.trivia)}`
2574
+ + `/s${context.scanSkip.map(parserId).join(',')}`
2575
+ + `/l${context.trackLines ? 1 : 0}`
2576
+ + `/c${context.captureTrivia ? 1 : 0}`
2577
+ + `/r${context.opaqueRootCapture ? 1 : 0}`
2578
+ + `/d${context.dynamicState ? 1 : 0}`
2579
+ const topPath = new Map<Combinator<unknown>, Map<string, string>>()
2580
+ const ownedPath = new Map<Combinator<unknown>, Map<string, string>>()
2581
+ const candidates = new Map<string, CapabilityCandidate>()
2582
+ const bindingEdges = new Map<string, BindingEdgeCandidate>()
2583
+ const contexts = new Map<string, CapabilityContext>()
2584
+ const visit = (parser: Combinator<unknown>, path: string, owned: boolean, context: CapabilityContext): void => {
2585
+ const contextKey = contextKeyOf(context)
2586
+ if (!contexts.has(contextKey)) contexts.set(contextKey, context)
2587
+ const seen = owned ? ownedPath : topPath
2588
+ let paths = seen.get(parser)
2589
+ if (paths === undefined) { paths = new Map(); seen.set(parser, paths) }
2590
+ const prior = paths.get(contextKey)
2591
+ if (prior !== undefined && prior <= path) return
2592
+ paths.set(contextKey, path)
2593
+
2594
+ const def = parser._def
2595
+ const recognitionContext = def.tag === 'token'
2596
+ ? { ...context, trivia: undefined, captureTrivia: false }
2597
+ : context
2598
+ const recognitionContextKey = contextKeyOf(recognitionContext)
2599
+ if (!owned && keyOf(def) !== undefined) {
2600
+ const key = `terminal\u0000${parserId(parser)}\u0000${contextKey}`
2601
+ const priorCandidate = candidates.get(key)
2602
+ if (priorCandidate === undefined || path < priorCandidate.path) {
2603
+ candidates.set(key, {
2604
+ bodyId: parserId(parser), parser, atom: 'terminal', path, context, contextKey, recognitionContextKey,
2605
+ })
2606
+ }
2607
+ } else if (!owned && (def.tag === 'token' || def.tag === 'choice' || def.tag === 'dispatch')) {
2608
+ const atom = def.tag
2609
+ const key = `${atom}\u0000${parserId(parser)}\u0000${contextKey}`
2610
+ const priorCandidate = candidates.get(key)
2611
+ if (priorCandidate === undefined || path < priorCandidate.path) {
2612
+ candidates.set(key, {
2613
+ bodyId: parserId(parser), parser, atom, path, context, contextKey, recognitionContextKey,
2614
+ })
2615
+ }
2616
+ }
2617
+ // Exact atomic ownership: the token's child is recognition machinery, not
2618
+ // a separately linked parser site or fixed parent-edge obligation.
2619
+ if (!owned && def.tag === 'token') return
2620
+ const childOwned = owned || def.tag === 'token'
2621
+ const children = tokenChildren(parser, resolve)
2622
+ let childContext = context
2623
+ if (def.tag === 'grammar') {
2624
+ childContext = {
2625
+ trivia: def.clearTrivia ? undefined : (def.triviaParser ?? context.trivia),
2626
+ scanSkip: context.scanSkip,
2627
+ trackLines: context.trackLines || def.trackLines,
2628
+ captureTrivia: context.captureTrivia || def.captureTrivia === true,
2629
+ opaqueRootCapture: context.opaqueRootCapture || def.rootCapture === 'opaque',
2630
+ dynamicState: context.dynamicState,
2631
+ }
2632
+ } else if (def.tag === 'withCtx') {
2633
+ childContext = { ...context, dynamicState: true }
2634
+ } else if (def.tag === 'token') {
2635
+ childContext = { ...context, trivia: undefined }
2636
+ }
2637
+ for (let i = 0; i < children.length; i++) {
2638
+ const child = children[i]!
2639
+ const childPath = `${path}/${String(i).padStart(4, '0')}`
2640
+ const edgeKey = `${parserId(parser)}\u0000${i}\u0000${contextKey}`
2641
+ if (!bindingEdges.has(edgeKey)) bindingEdges.set(edgeKey, {
2642
+ path: childPath,
2643
+ contextKey, childContextKey: contextKeyOf(childContext),
2644
+ parentTag: def.tag,
2645
+ childTag: child._def.tag,
2646
+ parentBodyId: parserId(parser),
2647
+ childBodyId: parserId(child),
2648
+ childOrdinal: i,
2649
+ })
2650
+ visit(child, childPath, childOwned, childContext)
2651
+ }
2652
+ }
2653
+ for (let i = 0; i < roots.length; i++) {
2654
+ const root = roots[i]!
2655
+ const rootPath = String(i).padStart(4, '0')
2656
+ const rootContext: CapabilityContext = {
2657
+ trivia: root._meta.grammarTrivia,
2658
+ scanSkip: root._meta.grammarScanSkip ?? [],
2659
+ trackLines: root._meta.grammarTrackLines === true,
2660
+ captureTrivia: false,
2661
+ opaqueRootCapture: false,
2662
+ dynamicState: false,
2663
+ }
2664
+ const contextKey = contextKeyOf(rootContext)
2665
+ bindingEdges.set(`root\u0000${i}\u0000${contextKey}`, {
2666
+ path: rootPath, contextKey, childContextKey: contextKey,
2667
+ parentTag: 'root', childTag: root._def.tag,
2668
+ parentBodyId: -1 - i, childBodyId: parserId(root), childOrdinal: i,
2669
+ })
2670
+ visit(root, rootPath, false, rootContext)
2671
+ }
2672
+ return {
2673
+ candidates: [...candidates.values()].sort((a, b) =>
2674
+ a.path < b.path ? -1 : a.path > b.path ? 1 : a.atom < b.atom ? -1 : a.atom > b.atom ? 1 : 0),
2675
+ bindingEdges: [...bindingEdges.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0),
2676
+ contexts,
2677
+ contextKeyOf,
2678
+ }
2679
+ }
2680
+
2681
+ function decisionViewKey(view: LexicalDecisionOutcomeView): string {
2682
+ if (view.kind === 'whole') return 'whole'
2683
+ if (view.kind === 'predicate') return `predicate\u0000${canonicalLexicalOutcomeKey(view.match)}`
2684
+ return `language\u0000${view.relation}\u0000${JSON.stringify(view.ir)}`
2685
+ }
2686
+
2687
+ function lexicalDecisionInventory(
2688
+ candidates: readonly CapabilityCandidate[],
2689
+ contextSnapshotIdByKey: ReadonlyMap<string, number>,
2690
+ resolve?: (name: string) => Combinator<unknown> | undefined,
2691
+ ): {
2692
+ families: LexicalDecisionFamily[]
2693
+ outcomes: LexicalDecisionOutcome[]
2694
+ decisions: LexicalDecisionSite[]
2695
+ recognitionBySite: ReadonlyMap<number, LexicalCapabilityStatus>
2696
+ } {
2697
+ const familyIrByKey = new Map<string, LexicalIr>()
2698
+ const viewsByKey = new Map<string, PendingDecisionView>()
2699
+ const pending: PendingDecisionSite[] = []
2700
+ const addFamily = (lead: DecisionLead): string => {
2701
+ if (!familyIrByKey.has(lead.key)) familyIrByKey.set(lead.key, lead.ir)
2702
+ return lead.key
2703
+ }
2704
+ const addView = (familyKey: string, view: LexicalDecisionOutcomeView): string => {
2705
+ const key = `${familyKey}\u0000${decisionViewKey(view)}`
2706
+ if (!viewsByKey.has(key)) viewsByKey.set(key, { familyKey, key, view })
2707
+ return key
2708
+ }
2709
+ const reuseEpochsForChoice = (
2710
+ arms: readonly Combinator<unknown>[],
2711
+ ): LexicalDecisionReuseEpoch[] => {
2712
+ const epochs: LexicalDecisionReuseEpoch[] = []
2713
+ let pure: number[] = []
2714
+ const flushPure = (): void => {
2715
+ if (pure.length === 0) return
2716
+ epochs.push({ armIds: pure, pendingReuse: 'unproved', boundary: 'none' })
2717
+ pure = []
2718
+ }
2719
+ for (let armId = 0; armId < arms.length; armId++) {
2720
+ if (!containsDynamicScanConsumer(arms[armId]!, resolve)) {
2721
+ pure.push(armId)
2722
+ continue
2723
+ }
2724
+ flushPure()
2725
+ epochs.push({ armIds: [armId], pendingReuse: 'forbidden', boundary: 'dynamic-scan' })
2726
+ }
2727
+ flushPure()
2728
+ return epochs
2729
+ }
2730
+
2731
+ // The interner is grammar-wide, not derived from whichever decisions happen
2732
+ // to consume a language. This keeps family identity independent of cost/site
2733
+ // selection and gives the legacy token inventory an explicit relocation.
2734
+ for (const candidate of candidates) {
2735
+ const def = candidate.parser._def
2736
+ if (candidate.atom !== 'terminal' && candidate.atom !== 'token') continue
2737
+ const normalized = normalizeBalancedLexical(candidate.parser, resolve)
2738
+ ?? normalizeLexical(def.tag === 'token' ? def.parser : candidate.parser, resolve, new Set())
2739
+ if (!('refusal' in normalized)) familyIrByKey.set(JSON.stringify(normalized.ir), normalized.ir)
2740
+ }
2741
+
2742
+ for (let siteId = 0; siteId < candidates.length; siteId++) {
2743
+ const candidate = candidates[siteId]!
2744
+ if (candidate.atom !== 'choice' && candidate.atom !== 'dispatch') continue
2745
+ const def = candidate.parser._def
2746
+ const precisionNotes: string[] = []
2747
+ const contextSnapshotId = contextSnapshotIdByKey.get(candidate.contextKey)
2748
+ if (contextSnapshotId === undefined) throw new Error('parseman: decision occurrence lost its context snapshot')
2749
+ const dynamicScan = candidate.atom === 'dispatch' && def.tag === 'dispatch'
2750
+ ? containsDynamicScanConsumer(def.selector, resolve)
2751
+ : containsDynamicScanConsumer(candidate.parser, resolve)
2752
+ if (dynamicScan) {
2753
+ precisionNotes.push('dynamic scan consumer forbids pending range reuse')
2754
+ }
2755
+ // A dispatch cannot classify before its selector when the selector itself is
2756
+ // a dynamic scan. Choice is different: a scan in one arm does not erase the
2757
+ // pure source-range languages of its siblings. Those arms remain
2758
+ // unrestricted and the SITE forbids pending reuse, but the compiler still
2759
+ // needs the sibling families to build and price a complete TOKEN body.
2760
+ if (candidate.atom === 'dispatch' && dynamicScan) {
2761
+ pending.push({
2762
+ siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
2763
+ contextSnapshotId, families: [], pendingReuse: 'forbidden',
2764
+ reuseEpochs: [{
2765
+ armIds: def.tag === 'dispatch'
2766
+ ? Array.from({ length: def.cases.length + (def.matchers?.length ?? 0)
2767
+ + (def.otherwise === undefined ? 0 : 1) }, (_, armId) => armId)
2768
+ : [],
2769
+ pendingReuse: 'forbidden', boundary: 'dynamic-scan',
2770
+ }],
2771
+ precisionNotes,
2772
+ })
2773
+ continue
2774
+ }
2775
+ if (candidate.atom === 'dispatch') {
2776
+ if (def.tag !== 'dispatch') throw new Error('parseman: decision occurrence lost its dispatch')
2777
+ const routeUsesRouted = [
2778
+ ...def.cases.map(branchUsesRouted),
2779
+ ...(def.matchers ?? []).map(branchUsesRouted),
2780
+ ...(def.otherwise === undefined ? [] : [branchUsesRouted({
2781
+ parser: def.otherwise, usesRouted: def.otherwiseUsesRouted,
2782
+ })]),
2783
+ ]
2784
+ const result = decisionLead(def.selector, candidate.context, resolve)
2785
+ if ('gap' in result) {
2786
+ precisionNotes.push(`selector: ${result.gap}`)
2787
+ pending.push({
2788
+ siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
2789
+ contextSnapshotId, families: [], precisionNotes, pendingReuse: 'unproved',
2790
+ reuseEpochs: [{
2791
+ armIds: Array.from({ length: def.cases.length + (def.matchers?.length ?? 0)
2792
+ + (def.otherwise === undefined ? 0 : 1) }, (_, armId) => armId),
2793
+ pendingReuse: 'unproved', boundary: 'none',
2794
+ }],
2795
+ routeUsesRouted,
2796
+ })
2797
+ continue
2798
+ }
2799
+ const familyKey = addFamily(result.lead)
2800
+ const predicates = predicateViews(def)
2801
+ const predicateKeys = predicates.map(({ match }) => addView(familyKey, {
2802
+ kind: 'predicate', relation: 'equal', match,
2803
+ }))
2804
+ const arms: Array<Omit<LexicalDecisionArm, 'acceptance'> & { acceptance: PendingAcceptance }> = []
2805
+ let offset = 0
2806
+ for (let armId = 0; armId < def.cases.length; armId++) {
2807
+ const entry = def.cases[armId]!
2808
+ const keys = predicateKeys.slice(offset, offset + entry.keys.length)
2809
+ offset += entry.keys.length
2810
+ arms.push({
2811
+ armId, acceptance: { kind: 'outcomes', keys },
2812
+ usesRouted: branchUsesRouted(entry), dynamicGate: false,
2813
+ })
2814
+ }
2815
+ for (let i = 0; i < (def.matchers?.length ?? 0); i++) {
2816
+ const entry = def.matchers![i]!
2817
+ arms.push({
2818
+ armId: arms.length, acceptance: { kind: 'outcomes', keys: [predicateKeys[offset++]!] },
2819
+ usesRouted: branchUsesRouted(entry), dynamicGate: false,
2820
+ })
2821
+ }
2822
+ if (def.otherwise !== undefined) arms.push({
2823
+ armId: arms.length,
2824
+ acceptance: { kind: 'otherwise', excludingKeys: predicateKeys },
2825
+ usesRouted: branchUsesRouted({ parser: def.otherwise, usesRouted: def.otherwiseUsesRouted }),
2826
+ dynamicGate: false,
2827
+ })
2828
+ pending.push({
2829
+ siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
2830
+ contextSnapshotId, families: [{ familyKey, arms }], precisionNotes,
2831
+ pendingReuse: dynamicScan ? 'forbidden' : 'unproved',
2832
+ reuseEpochs: [{
2833
+ armIds: arms.map(arm => arm.armId),
2834
+ pendingReuse: dynamicScan ? 'forbidden' : 'unproved',
2835
+ boundary: dynamicScan ? 'dynamic-scan' : 'none',
2836
+ }],
2837
+ routeUsesRouted,
2838
+ })
2839
+ continue
2840
+ }
2841
+
2842
+ if (def.tag !== 'choice') throw new Error('parseman: decision occurrence lost its choice')
2843
+ const leads = def.parsers.map((arm, armId) => {
2844
+ const result = decisionLead(arm, candidate.context, resolve)
2845
+ if ('gap' in result) precisionNotes.push(`arm ${armId}: ${result.gap}`)
2846
+ return result
2847
+ })
2848
+ const familyKeys = [...new Set(leads.flatMap(result => 'lead' in result
2849
+ ? [addFamily(result.lead)] : []))].sort()
2850
+ if (familyKeys.length === 0 && precisionNotes.length === 0) {
2851
+ precisionNotes.push('choice has no predecision range family; all arms remain unrestricted')
2852
+ }
2853
+ const families = familyKeys.map(familyKey => {
2854
+ const familyIr = familyIrByKey.get(familyKey)!
2855
+ const wholeKey = addView(familyKey, { kind: 'whole', relation: 'equal' })
2856
+ const familyDispatches = [...new Set(leads.flatMap(result =>
2857
+ 'lead' in result && result.lead.key === familyKey && result.lead.dispatch !== undefined
2858
+ ? [result.lead.dispatch] : []))]
2859
+ const ambiguousPartition = familyDispatches.length > 1
2860
+ if (ambiguousPartition) precisionNotes.push(
2861
+ `family ${familyKey}: multiple dispatches need distinct site-local outcome partitions`)
2862
+ const familyPredicates = ambiguousPartition ? []
2863
+ : familyDispatches.flatMap(dispatch => predicateViews(dispatch).map(entry => entry.match))
2864
+ const predicateKeys = familyPredicates.map(match => addView(familyKey, {
2865
+ kind: 'predicate', relation: 'equal', match,
2866
+ }))
2867
+ const arms = def.parsers.map((arm, armId): Omit<LexicalDecisionArm, 'acceptance'> & {
2868
+ acceptance: PendingAcceptance
2869
+ } => {
2870
+ const dynamicGate = def.gates[armId] !== null || def.autoNot[armId] !== null
2871
+ const result = leads[armId]!
2872
+ if ('gap' in result) return {
2873
+ armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate,
2874
+ }
2875
+ const lead = result.lead
2876
+ if (lead.key === familyKey) {
2877
+ if (ambiguousPartition) return {
2878
+ armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate,
2879
+ }
2880
+ if (lead.dispatch !== undefined) {
2881
+ const predicates = predicateViews(lead.dispatch)
2882
+ const keys = predicates.map(({ match }) => addView(familyKey, {
2883
+ kind: 'predicate', relation: 'equal', match,
2884
+ }))
2885
+ return {
2886
+ armId,
2887
+ acceptance: lead.dispatch.otherwise === undefined
2888
+ ? { kind: 'outcomes', keys }
2889
+ : { kind: 'unrestricted' },
2890
+ usesRouted: false,
2891
+ dynamicGate,
2892
+ }
2893
+ }
2894
+ return {
2895
+ armId, acceptance: { kind: 'outcomes', keys: [wholeKey] },
2896
+ usesRouted: false, dynamicGate,
2897
+ }
2898
+ }
2899
+ const familyLead = leads.find((entry): entry is { lead: DecisionLead } =>
2900
+ 'lead' in entry && entry.lead.key === familyKey)!.lead
2901
+ const armFirst = firstSetOf(arm, new Set(), resolve)
2902
+ const familyFirst = firstSetOf(familyLead.parser, new Set(), resolve)
2903
+ if (armFirst.kind === 'ranges' && familyFirst.kind === 'ranges'
2904
+ && !intersects(armFirst, familyFirst)) {
2905
+ return { armId, acceptance: { kind: 'impossible' }, usesRouted: false, dynamicGate }
2906
+ }
2907
+ const compatible = compatibleLanguageViews(lead.ir, familyIr)
2908
+ if (compatible !== undefined) {
2909
+ if (compatible.every(view => view.relation === 'equal')
2910
+ && familyPredicates.length > 0
2911
+ && predicatePartitionStableFor(lead.ir, familyPredicates)) {
2912
+ const values = finiteLexicalValues(lead.ir)!
2913
+ const accepted = new Set<string>()
2914
+ let hasOtherwise = false
2915
+ for (const value of values) {
2916
+ const matching = familyPredicates.flatMap((match, index) =>
2917
+ lexicalOutcomeMatches(match, value, 0, value.length) ? [predicateKeys[index]!] : [])
2918
+ if (matching.length === 0) hasOtherwise = true
2919
+ else for (const key of matching) accepted.add(key)
2920
+ }
2921
+ if (hasOtherwise && accepted.size === 0) return {
2922
+ armId,
2923
+ acceptance: { kind: 'otherwise', excludingKeys: predicateKeys },
2924
+ usesRouted: false,
2925
+ dynamicGate,
2926
+ }
2927
+ if (!hasOtherwise && accepted.size > 0) return {
2928
+ armId,
2929
+ acceptance: { kind: 'outcomes', keys: [...accepted] },
2930
+ usesRouted: false,
2931
+ dynamicGate,
2932
+ }
2933
+ // A finite arm split across direct predicates and the local
2934
+ // complement is conservatively retained; neither half is dropped.
2935
+ return { armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate }
2936
+ }
2937
+ const keys = compatible.map(view => addView(familyKey, {
2938
+ kind: 'language', relation: view.relation, ir: view.ir,
2939
+ }))
2940
+ return {
2941
+ armId, acceptance: { kind: 'outcomes', keys },
2942
+ usesRouted: false, dynamicGate,
2943
+ }
2944
+ }
2945
+ return { armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate }
2946
+ })
2947
+ return { familyKey, arms }
2948
+ })
2949
+ pending.push({
2950
+ siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
2951
+ contextSnapshotId, families, precisionNotes: [...new Set(precisionNotes)].sort(),
2952
+ pendingReuse: dynamicScan ? 'forbidden' : 'unproved',
2953
+ reuseEpochs: reuseEpochsForChoice(def.parsers),
2954
+ })
2955
+ }
2956
+
2957
+ const familyKeys = [...familyIrByKey.keys()].sort()
2958
+ const familyIdByKey = new Map(familyKeys.map((key, index) => [key, FIRST_LEXICAL_FAMILY_ID + index]))
2959
+ const families = familyKeys.map((semanticKey, index): LexicalDecisionFamily => ({
2960
+ id: FIRST_LEXICAL_FAMILY_ID + index, semanticKey, ir: familyIrByKey.get(semanticKey)!,
2961
+ }))
2962
+ const viewKeys = [...viewsByKey.keys()].sort()
2963
+ const firstOutcomeId = FIRST_LEXICAL_FAMILY_ID + familyKeys.length
2964
+ const outcomeIdByKey = new Map(viewKeys.map((key, index) => [key, firstOutcomeId + index]))
2965
+ const outcomes = viewKeys.map((key, index): LexicalDecisionOutcome => {
2966
+ const pendingView = viewsByKey.get(key)!
2967
+ return {
2968
+ id: firstOutcomeId + index,
2969
+ familyId: familyIdByKey.get(pendingView.familyKey)!,
2970
+ semanticKey: decisionViewKey(pendingView.view),
2971
+ view: pendingView.view,
2972
+ }
2973
+ })
2974
+ const decisions = pending.map((site): LexicalDecisionSite => ({
2975
+ siteId: site.siteId, atom: site.atom, path: site.path, contextKey: site.contextKey,
2976
+ contextSnapshotId: site.contextSnapshotId,
2977
+ fallback: 'unrestricted', precisionNotes: site.precisionNotes, pendingReuse: site.pendingReuse,
2978
+ reuseEpochs: site.reuseEpochs,
2979
+ ...(site.routeUsesRouted === undefined ? {} : { routeUsesRouted: site.routeUsesRouted }),
2980
+ families: site.families.map(family => ({
2981
+ familyId: familyIdByKey.get(family.familyKey)!,
2982
+ arms: family.arms.map(arm => ({
2983
+ ...arm,
2984
+ acceptance: arm.acceptance.kind === 'outcomes'
2985
+ ? { kind: 'outcomes', outcomeIds: [...new Set(arm.acceptance.keys.map(key => outcomeIdByKey.get(key)!))].sort((a, b) => a - b) }
2986
+ : arm.acceptance.kind === 'otherwise'
2987
+ ? { kind: 'otherwise', excludingOutcomeIds: [...new Set(arm.acceptance.excludingKeys.map(key => outcomeIdByKey.get(key)!))].sort((a, b) => a - b) }
2988
+ : arm.acceptance,
2989
+ })),
2990
+ })),
2991
+ }))
2992
+ const recognitionBySite = new Map<number, LexicalCapabilityStatus>(decisions.map(site => [
2993
+ site.siteId, COMPLETE_CAPABILITY,
2994
+ ]))
2995
+ return { families, outcomes, decisions, recognitionBySite }
2996
+ }
2997
+
2998
+ function classifiedLiteralProjection(parser: Combinator<unknown>): {
2999
+ spelling: string
3000
+ transforms: readonly {
3001
+ fn: (value: unknown, span: { start: number; end: number }) => unknown
3002
+ source?: string
3003
+ }[]
3004
+ } | undefined {
3005
+ let current = parser
3006
+ const outerFirst: Array<{
3007
+ fn: (value: unknown, span: { start: number; end: number }) => unknown
3008
+ source?: string
3009
+ }> = []
3010
+ const seen = new Set<Combinator<unknown>>()
3011
+ while (!seen.has(current)) {
3012
+ seen.add(current)
3013
+ const def = current._def
3014
+ if (def.tag === 'literal') return { spelling: def.value, transforms: outerFirst.reverse() }
3015
+ if (def.tag === 'transform') {
3016
+ outerFirst.push({ fn: def.fn, ...(def.fnSrc === undefined ? {} : { source: def.fnSrc }) })
3017
+ current = def.parser
3018
+ continue
3019
+ }
3020
+ if (def.tag === 'label') { current = def.parser; continue }
3021
+ return undefined
3022
+ }
3023
+ return undefined
3024
+ }
3025
+
3026
+ function decisionEffectInventory(
3027
+ candidates: readonly CapabilityCandidate[],
3028
+ decisions: readonly LexicalDecisionSite[],
3029
+ bindingEdges: readonly LexicalBindingEdge[],
3030
+ bindingProjections: readonly LexicalBindingProjection[],
3031
+ resolve?: (name: string) => Combinator<unknown> | undefined,
3032
+ ): LexicalDecisionEffectInventory {
3033
+ const finalDecisionAuthorities: LexicalFinalDecisionAuthority[] = []
3034
+ const decisionEffects: LexicalDecisionEffectProgram[] = []
3035
+ const decisionChildEffects: LexicalDecisionChildEffect[] = []
3036
+ const choiceClassAuthorities: LexicalChoiceClassAuthority[] = []
3037
+ const choiceExecutionOrders: LexicalChoiceExecutionOrder[] = []
3038
+ const classifySpellingMaps: LexicalClassifySpellingMap[] = []
3039
+ const classifyProjectionPlans: LexicalClassifyProjectionPlan[] = []
3040
+ const decisionExpectedAuthorities: LexicalExpectedAuthority[] = []
3041
+ const decisionCallbackAuthorities: LexicalDecisionCallbackAuthority[] = []
3042
+ const expectedIdByKey = new Map<string, number>()
3043
+ const callbackIds = new Map<LexicalDecisionCallbackAuthority['callback'], number>()
3044
+ const sourceIds = new Map<string, number>()
3045
+ const autoNotIds = new Map<string, number>()
3046
+ const edgeByPath = new Map(bindingEdges.map(edge => [edge.path, edge.id]))
3047
+ const expectedId = (values: readonly string[]): number => {
3048
+ const key = JSON.stringify(values)
3049
+ const prior = expectedIdByKey.get(key)
3050
+ if (prior !== undefined) return prior
3051
+ const id = decisionExpectedAuthorities.length
3052
+ expectedIdByKey.set(key, id)
3053
+ decisionExpectedAuthorities.push({ id, values: [...values] })
3054
+ return id
3055
+ }
3056
+ const callbackId = (value: LexicalDecisionCallbackAuthority['callback']): number => {
3057
+ const prior = callbackIds.get(value)
3058
+ if (prior !== undefined) return prior
3059
+ const id = decisionCallbackAuthorities.length
3060
+ callbackIds.set(value, id)
3061
+ decisionCallbackAuthorities.push({ id, callback: value })
3062
+ return id
3063
+ }
3064
+ const stringId = (map: Map<string, number>, value: string): number => {
3065
+ const prior = map.get(value)
3066
+ if (prior !== undefined) return prior
3067
+ const id = map.size
3068
+ map.set(value, id)
3069
+ return id
3070
+ }
3071
+ const edgeAt = (path: string, childIndex: number): number => {
3072
+ const childPath = `${path}/${String(childIndex).padStart(4, '0')}`
3073
+ const edge = edgeByPath.get(childPath)
3074
+ if (edge === undefined) throw new Error(`parseman: decision child lost binding edge ${childPath}`)
3075
+ return edge
3076
+ }
3077
+ const childEffect = (
3078
+ siteId: number,
3079
+ armId: number,
3080
+ childBindingEdgeId: number,
3081
+ parser: Combinator<unknown>,
3082
+ gate: ((state: unknown) => boolean) | null = null,
3083
+ gateSource: string | null = null,
3084
+ autoNot: readonly AutoNotCheck[] | null = null,
3085
+ ): number => {
3086
+ const id = decisionChildEffects.length
3087
+ decisionChildEffects.push({
3088
+ id, decisionSiteId: siteId, armId, childBindingEdgeId,
3089
+ expectedAuthorityId: expectedId(deriveExpected(parser)),
3090
+ ...(gate === null ? {} : { gateCallbackId: callbackId(gate) }),
3091
+ ...(gateSource === null ? {} : { gateSourceId: stringId(sourceIds, gateSource) }),
3092
+ ...(autoNot === null ? {} : { autoNotAuthorityId: stringId(autoNotIds, JSON.stringify(autoNot)) }),
3093
+ })
3094
+ return id
3095
+ }
3096
+
3097
+ for (const decision of decisions) {
3098
+ const candidate = candidates[decision.siteId]
3099
+ if (candidate === undefined || candidate.parser._def.tag !== decision.atom) {
3100
+ throw new Error(`parseman: decision effect lost occurrence ${decision.siteId}`)
3101
+ }
3102
+ const def = candidate.parser._def
3103
+ const authorityId = finalDecisionAuthorities.length
3104
+ const effectId = decisionEffects.length
3105
+ let loweringGap: string | undefined
3106
+ const childEffectIds: number[] = []
3107
+ if (def.tag === 'choice') {
3108
+ const finalChoice = classifyFinalChoice(def.parsers, resolve)
3109
+ const authoredArmIds = def.parsers.map((_, armId) => armId)
3110
+ let mode: Extract<LexicalFinalDecisionAuthority, { atom: 'choice' }>['mode']
3111
+ if (finalChoice.exclusive) {
3112
+ const classAuthorityId = choiceClassAuthorities.length
3113
+ choiceClassAuthorities.push({
3114
+ id: classAuthorityId, armIds: authoredArmIds, firstSets: finalChoice.firstSets,
3115
+ })
3116
+ mode = { kind: 'exclusive', classAuthorityId }
3117
+ } else if (def.strategy.tag === 'literalsLongestFirst') {
3118
+ const orderAuthorityId = choiceExecutionOrders.length
3119
+ choiceExecutionOrders.push({
3120
+ id: orderAuthorityId, armIds: [...def.strategy.sortedIndices],
3121
+ })
3122
+ mode = { kind: 'longest', orderAuthorityId }
3123
+ } else if (def.strategy.tag === 'greedyClassify') {
3124
+ const spellingMapAuthorityId = classifySpellingMaps.length
3125
+ const projectionAuthorityId = classifyProjectionPlans.length
3126
+ const spellingEntries: Array<{ spelling: string; armId: number }> = []
3127
+ const projectionEntries: Array<{
3128
+ armId: number
3129
+ transformIds: readonly number[]
3130
+ transformSourceIds: readonly (number | -1)[]
3131
+ transformCount: number
3132
+ }> = []
3133
+ for (let armId = 0; armId < def.parsers.length; armId++) {
3134
+ if (armId === def.strategy.superIndex) continue
3135
+ const projection = classifiedLiteralProjection(def.parsers[armId]!)
3136
+ if (projection === undefined) {
3137
+ throw new Error(`parseman: greedy classifier lost literal arm ${armId}`)
3138
+ }
3139
+ spellingEntries.push({ spelling: projection.spelling, armId })
3140
+ const transformIds = projection.transforms.map(entry => callbackId(entry.fn))
3141
+ const transformSourceIds = projection.transforms.map(entry =>
3142
+ entry.source === undefined ? -1 : stringId(sourceIds, entry.source))
3143
+ projectionEntries.push({
3144
+ armId, transformIds, transformSourceIds, transformCount: transformIds.length,
3145
+ })
3146
+ if (transformSourceIds.includes(-1)) {
3147
+ loweringGap = 'classified transform has no statically named reader projection'
3148
+ }
3149
+ }
3150
+ classifySpellingMaps.push({ id: spellingMapAuthorityId, entries: spellingEntries })
3151
+ classifyProjectionPlans.push({ id: projectionAuthorityId, entries: projectionEntries })
3152
+ mode = {
3153
+ kind: 'classify', superArmId: def.strategy.superIndex,
3154
+ spellingMapAuthorityId, projectionAuthorityId,
3155
+ }
3156
+ } else mode = { kind: 'ordered' }
3157
+ const expectedParsers = mode.kind === 'longest'
3158
+ ? mode.orderAuthorityId === undefined ? def.parsers
3159
+ : choiceExecutionOrders[mode.orderAuthorityId]!.armIds.map(armId => def.parsers[armId]!)
3160
+ : mode.kind === 'classify' ? [def.parsers[mode.superArmId]!]
3161
+ : mode.kind === 'exclusive' ? def.parsers : undefined
3162
+ finalDecisionAuthorities.push({
3163
+ id: authorityId, atom: 'choice', decisionSiteId: decision.siteId,
3164
+ authoredArmIds, mode,
3165
+ ...(expectedParsers === undefined ? {} : {
3166
+ staticMissExpectedAuthorityId: expectedId(
3167
+ expectedParsers.flatMap(parser => deriveExpected(parser)),
3168
+ ),
3169
+ }),
3170
+ })
3171
+ for (let armId = 0; armId < def.parsers.length; armId++) {
3172
+ const gate = def.gates[armId] ?? null
3173
+ const gateSource = def.gateSrcs?.[armId] ?? null
3174
+ if (gate !== null && gateSource === null) {
3175
+ loweringGap = 'dynamic choice gate has no statically named reader projection'
3176
+ }
3177
+ childEffectIds.push(childEffect(
3178
+ decision.siteId, armId, edgeAt(candidate.path, armId), def.parsers[armId]!,
3179
+ gate, gateSource, def.autoNot[armId],
3180
+ ))
3181
+ }
3182
+ } else {
3183
+ const routeParsers = [
3184
+ ...def.cases.map(entry => entry.parser),
3185
+ ...(def.matchers ?? []).map(entry => entry.parser),
3186
+ ...(def.otherwise === undefined ? [] : [def.otherwise]),
3187
+ ]
3188
+ const routeBranches = [
3189
+ ...def.cases,
3190
+ ...(def.matchers ?? []),
3191
+ ...(def.otherwise === undefined ? [] : [{
3192
+ parser: def.otherwise, usesRouted: def.otherwiseUsesRouted,
3193
+ }]),
3194
+ ]
3195
+ const otherwiseOffset = def.otherwise === undefined ? 0 : 1
3196
+ const routeEdgeIndices = [
3197
+ ...def.cases.map((_, index) => 1 + otherwiseOffset + index),
3198
+ ...(def.matchers ?? []).map((_, index) => 1 + otherwiseOffset + def.cases.length + index),
3199
+ ...(def.otherwise === undefined ? [] : [1]),
3200
+ ]
3201
+ const selectorBindingEdgeId = edgeAt(candidate.path, 0)
3202
+ const orderedRouteIds = routeParsers.map((_, routeId) => routeId)
3203
+ finalDecisionAuthorities.push({
3204
+ id: authorityId, atom: 'dispatch', decisionSiteId: decision.siteId,
3205
+ selectorBindingEdgeId,
3206
+ // The Stage-A occurrence owns exact/fold/matcher/otherwise precedence.
3207
+ classifierAuthorityId: decision.siteId,
3208
+ orderedRouteIds,
3209
+ noRouteExpectedAuthorityId: expectedId(def.cases.flatMap(entry =>
3210
+ entry.keys.map(key => JSON.stringify(key)))),
3211
+ })
3212
+ for (let routeId = 0; routeId < routeParsers.length; routeId++) {
3213
+ childEffectIds.push(childEffect(
3214
+ decision.siteId, routeId, edgeAt(candidate.path, routeEdgeIndices[routeId]!),
3215
+ routeParsers[routeId]!,
3216
+ ))
3217
+ }
3218
+ const routedAuthority = decision.routeUsesRouted
3219
+ if (routedAuthority === undefined || routedAuthority.length !== routeBranches.length
3220
+ || routedAuthority.some((value, index) => value !== branchUsesRouted(routeBranches[index]!))) {
3221
+ loweringGap = 'dispatch routed policy lost its Stage-A authority'
3222
+ }
3223
+ }
3224
+ const childBindingProjectionIds = childEffectIds.map(childEffectId => {
3225
+ const child = decisionChildEffects[childEffectId]
3226
+ if (child === undefined) throw new Error('parseman: decision effect lost its child effect')
3227
+ const edge = bindingEdges[child.childBindingEdgeId]
3228
+ if (edge?.projectionId === undefined) {
3229
+ throw new Error('parseman: decision effect lost its child binding projection')
3230
+ }
3231
+ return edge.projectionId
3232
+ })
3233
+ let readerMask = ALL_BINDING_READERS
3234
+ let variantMask = ALL_BINDING_VARIANTS
3235
+ for (const projectionId of childBindingProjectionIds) {
3236
+ const projection = bindingProjections[projectionId]
3237
+ if (projection === undefined) throw new Error('parseman: decision binding projection disappeared')
3238
+ readerMask &= projection.readerMask
3239
+ variantMask &= projection.variantMask
3240
+ }
3241
+ if (childBindingProjectionIds.length === 0) {
3242
+ readerMask = 0
3243
+ variantMask = 0
3244
+ }
3245
+ if (loweringGap === undefined
3246
+ && (readerMask !== ALL_BINDING_READERS || variantMask !== ALL_BINDING_VARIANTS)) {
3247
+ loweringGap = 'decision child binding projection is incomplete'
3248
+ }
3249
+ const authority = finalDecisionAuthorities[authorityId]
3250
+ if (authority === undefined) throw new Error('parseman: decision effect lost its final authority')
3251
+ const digestWords = [
3252
+ decision.siteId, authorityId,
3253
+ authority.atom === 'choice' ? 0 : 1,
3254
+ DECISION_REFERENCE_TEMPLATE_ID, DECISION_CAPTURED_TEMPLATE_ID, DECISION_NAMED_TEMPLATE_ID,
3255
+ readerMask, variantMask, childEffectIds.length,
3256
+ ...(decision.routeUsesRouted ?? []).map(value => value ? 1 : 0),
3257
+ ...childEffectIds.flatMap((childEffectId, index) => {
3258
+ const child = decisionChildEffects[childEffectId]!
3259
+ const projection = bindingProjections[childBindingProjectionIds[index]!]!
3260
+ return [
3261
+ child.armId, child.expectedAuthorityId,
3262
+ child.gateCallbackId ?? -1, child.gateSourceId ?? -1, child.autoNotAuthorityId ?? -1,
3263
+ projection.id, projection.semanticDigest,
3264
+ ]
3265
+ }),
3266
+ ]
3267
+ decisionEffects.push({
3268
+ id: effectId, decisionSiteId: decision.siteId, finalDecisionAuthorityId: authorityId,
3269
+ phase: loweringGap === undefined ? COMPLETE_PHASE
3270
+ : phaseGap(COMPLETE_CAPABILITY, loweringGap),
3271
+ childEffectIds, childBindingProjectionIds,
3272
+ referenceTemplateId: DECISION_REFERENCE_TEMPLATE_ID,
3273
+ capturedTemplateId: DECISION_CAPTURED_TEMPLATE_ID,
3274
+ namedTemplateId: DECISION_NAMED_TEMPLATE_ID,
3275
+ readerMask, variantMask,
3276
+ semanticDigest: bindingDigest(digestWords, authority.atom, String(decision.siteId)),
3277
+ pendingRollbackPolicyId: 0, commitPolicyId: 0,
3278
+ })
3279
+ }
3280
+ return {
3281
+ finalDecisionAuthorities, decisionEffects, decisionChildEffects,
3282
+ choiceClassAuthorities, choiceExecutionOrders, classifySpellingMaps,
3283
+ classifyProjectionPlans, decisionExpectedAuthorities,
3284
+ decisionCallbackAuthorities,
3285
+ decisionSourceAuthorities: [...sourceIds.entries()].map(([source, id]) => ({ id, source }))
3286
+ .sort((a, b) => a.id - b.id),
3287
+ decisionAutoNotAuthorities: [...autoNotIds.entries()].map(([key, id]) => ({
3288
+ id, checks: JSON.parse(key) as AutoNotCheck[],
3289
+ })).sort((a, b) => a.id - b.id),
3290
+ }
3291
+ }
3292
+
3293
+ function lexicalCapabilityInventory(
3294
+ roots: ReadonlyArray<Combinator<unknown>>,
3295
+ resolve?: (name: string) => Combinator<unknown> | undefined,
3296
+ ): {
3297
+ capabilities: LexicalCapabilitySite[]
3298
+ languages: LexicalCapabilityLanguage[]
3299
+ bindingEdges: LexicalBindingEdge[]
3300
+ bindingProjections: LexicalBindingProjection[]
3301
+ terminalProjections: LexicalTerminalProjection[]
3302
+ decisionFamilies: LexicalDecisionFamily[]
3303
+ decisionOutcomes: LexicalDecisionOutcome[]
3304
+ decisions: LexicalDecisionSite[]
3305
+ transitionDiagnostics: LexicalTransitionDiagnosticPlan[]
3306
+ controlPlans: LexicalControlPlan[]
3307
+ boundaryPlans: LexicalBoundaryPlan[]
3308
+ materializationPlans: LexicalMaterializationPlan[]
3309
+ grammarWrapperSpecs: LexicalGrammarWrapperSpec[]
3310
+ grammarCaptureTriviaKinds: string[][]
3311
+ boundaryTopologies: LexicalBoundaryTopology[]
3312
+ contextSnapshots: LexicalContextSnapshot[]
3313
+ ownSkipPlans: LexicalOwnSkipPlan[]
3314
+ scanConsumerPolicies: LexicalScanConsumerPolicy[]
3315
+ decisionEffectPlan: LexicalDecisionEffectInventory
3316
+ } {
3317
+ const inventory = lexicalCapabilityCandidates(roots, resolve)
3318
+ const occurrenceContexts = new Map<Combinator<unknown>, Set<string>>()
3319
+ for (const candidate of inventory.candidates) {
3320
+ let contexts = occurrenceContexts.get(candidate.parser)
3321
+ if (contexts === undefined) occurrenceContexts.set(candidate.parser, contexts = new Set())
3322
+ contexts.add(candidate.contextKey)
3323
+ }
3324
+ const isSeparateOccurrence = (parser: Combinator<unknown>, contextKey: string): boolean =>
3325
+ occurrenceContexts.get(parser)?.has(contextKey) === true
3326
+ const scanConsumersBySite = new Map<number, ScanConsumerCandidate[]>()
3327
+ for (let siteId = 0; siteId < inventory.candidates.length; siteId++) {
3328
+ const candidate = inventory.candidates[siteId]!
3329
+ const consumers = ownedScanConsumers(
3330
+ candidate.parser, candidate.context, inventory.contextKeyOf, isSeparateOccurrence, resolve,
3331
+ )
3332
+ if (consumers.length === 0) continue
3333
+ scanConsumersBySite.set(siteId, consumers)
3334
+ for (const consumer of consumers) {
3335
+ if (!inventory.contexts.has(consumer.contextKey)) {
3336
+ inventory.contexts.set(consumer.contextKey, consumer.context)
3337
+ }
3338
+ }
3339
+ }
3340
+ const contextSnapshots = [...inventory.contexts.entries()]
3341
+ .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)
3342
+ .map(([key, context], id): LexicalContextSnapshot => ({
3343
+ id, key,
3344
+ hasTrivia: context.trivia !== undefined,
3345
+ hasScanSkip: context.scanSkip.length > 0,
3346
+ trackLines: context.trackLines,
3347
+ captureTrivia: context.captureTrivia,
3348
+ opaqueRootCapture: context.opaqueRootCapture,
3349
+ dynamicState: context.dynamicState,
3350
+ }))
3351
+ const contextSnapshotIdByKey = new Map(contextSnapshots.map(snapshot => [snapshot.key, snapshot.id]))
3352
+ const decisionInventory = lexicalDecisionInventory(
3353
+ inventory.candidates, contextSnapshotIdByKey, resolve,
3354
+ )
3355
+ const transitionDiagnostics: LexicalTransitionDiagnosticPlan[] = []
3356
+ const transitionDiagnosticIdByKey = new Map<string, number>()
3357
+ const internTransitionDiagnostics = (
3358
+ recognizerKey: string,
3359
+ session: LexicalNormalizeSession,
3360
+ ): number => {
3361
+ const body = {
3362
+ stateCount: session.nextState, expected: session.expected, events: session.events,
3363
+ }
3364
+ // State ids are local to the canonical recognizer transition graph. Equal
3365
+ // event rows over a different graph are not one plan merely by coincidence.
3366
+ const key = `${recognizerKey}\u0000${JSON.stringify(body)}`
3367
+ const prior = transitionDiagnosticIdByKey.get(key)
3368
+ if (prior !== undefined) return prior
3369
+ const id = transitionDiagnostics.length
3370
+ transitionDiagnosticIdByKey.set(key, id)
3371
+ transitionDiagnostics.push({ id, ...body })
3372
+ return id
3373
+ }
3374
+ const controlPlans: LexicalControlPlan[] = []
3375
+ const controlPlanIdByKey = new Map<string, number>()
3376
+ const internControlPlan = (session: LexicalNormalizeSession): number => {
3377
+ const key = JSON.stringify(session.controls)
3378
+ const prior = controlPlanIdByKey.get(key)
3379
+ if (prior !== undefined) return prior
3380
+ const id = controlPlans.length
3381
+ controlPlanIdByKey.set(key, id)
3382
+ controlPlans.push({ id, controls: session.controls })
3383
+ return id
3384
+ }
3385
+ const grammarWrapperSpecs: LexicalGrammarWrapperSpec[] = []
3386
+ const grammarSpecIdByKey = new Map<string, number>()
3387
+ const grammarSourceOperationIds = new Map<Combinator<unknown>, number>()
3388
+ const triviaBindingIds = new Map<Combinator<unknown>, number>()
3389
+ const grammarCaptureTriviaKinds: string[][] = []
3390
+ const captureKindsIdByKey = new Map<string, number>()
3391
+ const boundaryTopologies: LexicalBoundaryTopology[] = []
3392
+ const topologyIdByKey = new Map<string, number>()
3393
+ const identityId = (map: Map<Combinator<unknown>, number>, parser: Combinator<unknown>): number => {
3394
+ const prior = map.get(parser)
3395
+ if (prior !== undefined) return prior
3396
+ const id = map.size
3397
+ map.set(parser, id)
3398
+ return id
3399
+ }
3400
+ const captureKindsId = (values: readonly string[] | undefined): number | undefined => {
3401
+ if (values === undefined) return undefined
3402
+ const key = JSON.stringify(values)
3403
+ const prior = captureKindsIdByKey.get(key)
3404
+ if (prior !== undefined) return prior
3405
+ const id = grammarCaptureTriviaKinds.length
3406
+ captureKindsIdByKey.set(key, id)
3407
+ grammarCaptureTriviaKinds.push([...values])
3408
+ return id
3409
+ }
3410
+ const internGrammarWrapper = (
3411
+ parser: Combinator<unknown>,
3412
+ captureTriviaKinds: readonly string[] | undefined,
3413
+ ): number | undefined => {
3414
+ const def = parser._def
3415
+ if (def.tag !== 'grammar') return undefined
3416
+ if (def.constructionTrackLines === undefined) return undefined
3417
+ const sourceOperationId = identityId(grammarSourceOperationIds, parser)
3418
+ const triviaBindingId = def.triviaParser === undefined
3419
+ ? undefined : identityId(triviaBindingIds, def.triviaParser)
3420
+ const captureTriviaKindsId = captureKindsId(captureTriviaKinds)
3421
+ const body = {
3422
+ sourceOperationId,
3423
+ clearTrivia: def.clearTrivia === true,
3424
+ ...(triviaBindingId === undefined ? {} : { triviaBindingId }),
3425
+ trackLines: def.constructionTrackLines,
3426
+ captureTrivia: def.captureTrivia === true,
3427
+ ...(captureTriviaKindsId === undefined ? {} : { captureTriviaKindsId }),
3428
+ rootCapture: def.rootCapture === 'opaque' ? 'opaque' as const : 'inherit' as const,
3429
+ clonePolicy: 'spread-existing-or-create-canonical' as const,
3430
+ postChildPolicy: 'line-propagate-then-annotate-or-return' as const,
3431
+ }
3432
+ const key = JSON.stringify(body)
3433
+ const prior = grammarSpecIdByKey.get(key)
3434
+ if (prior !== undefined) return prior
3435
+ const id = grammarWrapperSpecs.length
3436
+ grammarSpecIdByKey.set(key, id)
3437
+ grammarWrapperSpecs.push({ id, ...body })
3438
+ return id
3439
+ }
3440
+ const internBoundaryTopology = (
3441
+ session: LexicalNormalizeSession,
3442
+ ): { readonly id: number } | { readonly gap: string } => {
3443
+ // Validate the entire overlay before interning any source-operation spec.
3444
+ // A declined late frame must not leave unreachable compiler metadata behind.
3445
+ const grammarCaptureKindsByFrame = new Map<number, readonly string[] | undefined>()
3446
+ for (let frameIndex = 0; frameIndex < session.wrapperFrames.length; frameIndex++) {
3447
+ const pending = session.wrapperFrames[frameIndex]!
3448
+ const control = session.controls[pending.controlId]
3449
+ if (pending.stateStart < 0 || pending.stateEnd < pending.stateStart
3450
+ || pending.stateEnd > session.nextState
3451
+ || pending.id !== frameIndex || control === undefined
3452
+ || control.id !== pending.controlId
3453
+ || control.kind !== pending.kind
3454
+ || control.stateStart !== pending.stateStart
3455
+ || control.stateEnd !== pending.stateEnd) {
3456
+ return { gap: 'token boundary overlay has an invalid state interval' }
3457
+ }
3458
+ if (pending.parentFrameId !== undefined) {
3459
+ const parent = session.wrapperFrames[pending.parentFrameId]
3460
+ if (parent === undefined || pending.parentFrameId >= pending.id
3461
+ || pending.stateStart < parent.stateStart || pending.stateEnd > parent.stateEnd) {
3462
+ return { gap: 'token boundary overlay is not well nested' }
3463
+ }
3464
+ }
3465
+ if (pending.kind === 'grammar') {
3466
+ const def = pending.parser._def
3467
+ if (def.tag !== 'grammar' || def.constructionTrackLines === undefined) {
3468
+ return { gap: 'grammar wrapper lacks exact construction-time source-operation metadata' }
3469
+ }
3470
+ try {
3471
+ grammarCaptureKindsByFrame.set(
3472
+ pending.id,
3473
+ def.constructionCaptureTriviaKinds === undefined
3474
+ ? undefined : [...def.constructionCaptureTriviaKinds],
3475
+ )
3476
+ } catch {
3477
+ return { gap: 'grammar wrapper capture-trivia policy cannot be snapshotted safely' }
3478
+ }
3479
+ }
3480
+ }
3481
+ for (let i = 0; i < session.wrapperFrames.length; i++) {
3482
+ for (let j = i + 1; j < session.wrapperFrames.length; j++) {
3483
+ const a = session.wrapperFrames[i]!
3484
+ const b = session.wrapperFrames[j]!
3485
+ const overlap = a.stateStart < b.stateEnd && b.stateStart < a.stateEnd
3486
+ const nested = (a.stateStart <= b.stateStart && b.stateEnd <= a.stateEnd)
3487
+ || (b.stateStart <= a.stateStart && a.stateEnd <= b.stateEnd)
3488
+ if (overlap && !nested) return { gap: 'token boundary overlay has crossed state intervals' }
3489
+ }
3490
+ }
3491
+ const outer = session.wrapperFrames[0]
3492
+ if (outer?.kind !== 'token' || outer.parentFrameId !== undefined
3493
+ || outer.stateStart !== 0 || outer.stateEnd !== session.nextState) {
3494
+ return { gap: 'token boundary overlay lost its outer authored token frame' }
3495
+ }
3496
+
3497
+ const frames: LexicalWrapperFrame[] = []
3498
+ for (const pending of session.wrapperFrames) {
3499
+ if (pending.kind === 'token') frames.push({
3500
+ id: pending.id,
3501
+ ...(pending.parentFrameId === undefined ? {} : { parentFrameId: pending.parentFrameId }),
3502
+ controlId: pending.controlId,
3503
+ stateStart: pending.stateStart,
3504
+ stateEnd: pending.stateEnd,
3505
+ kind: 'token', boundaryPlanId: 0, materializationPlanId: 0,
3506
+ })
3507
+ else {
3508
+ const wrapperSpecId = internGrammarWrapper(
3509
+ pending.parser, grammarCaptureKindsByFrame.get(pending.id),
3510
+ )
3511
+ if (wrapperSpecId === undefined) throw new Error('parseman: validated grammar policy disappeared')
3512
+ frames.push({
3513
+ id: pending.id,
3514
+ ...(pending.parentFrameId === undefined ? {} : { parentFrameId: pending.parentFrameId }),
3515
+ controlId: pending.controlId,
3516
+ stateStart: pending.stateStart,
3517
+ stateEnd: pending.stateEnd,
3518
+ kind: 'grammar', wrapperSpecId,
3519
+ })
3520
+ }
3521
+ }
3522
+ const key = JSON.stringify(frames)
3523
+ const prior = topologyIdByKey.get(key)
3524
+ if (prior !== undefined) return { id: prior }
3525
+ const id = boundaryTopologies.length
3526
+ topologyIdByKey.set(key, id)
3527
+ boundaryTopologies.push({ id, frames })
3528
+ return { id }
3529
+ }
3530
+ const ownSkipPlans: LexicalOwnSkipPlan[] = []
3531
+ const ownSkipPlanIdByKey = new Map<string, number>()
3532
+ const internOwnSkipPlan = (entries: readonly Combinator<unknown>[]): number | undefined => {
3533
+ if (entries.length === 0) return undefined
3534
+ const represented = entries.map((entry): LexicalOwnSkipEntry => {
3535
+ const normalized = normalizeBalancedLexical(entry, resolve)
3536
+ ?? normalizeLexical(entry, resolve, new Set())
3537
+ if ('refusal' in normalized) return {
3538
+ status: gap(`ownSkip normalization: ${normalized.refusal}`),
3539
+ }
3540
+ return {
3541
+ semanticKey: JSON.stringify(normalized.ir), ir: normalized.ir, status: COMPLETE_CAPABILITY,
3542
+ }
3543
+ })
3544
+ const key = JSON.stringify(represented)
3545
+ const prior = ownSkipPlanIdByKey.get(key)
3546
+ if (prior !== undefined) return prior
3547
+ const id = ownSkipPlans.length
3548
+ ownSkipPlanIdByKey.set(key, id)
3549
+ ownSkipPlans.push({ id, entries: represented })
3550
+ return id
3551
+ }
3552
+ const scanConsumerPolicies: LexicalScanConsumerPolicy[] = []
3553
+ const scanPolicyIdsBySite = new Map<number, number[]>()
3554
+ for (let siteId = 0; siteId < inventory.candidates.length; siteId++) {
3555
+ const candidate = inventory.candidates[siteId]!
3556
+ for (const consumer of scanConsumersBySite.get(siteId) ?? []) {
3557
+ const contextSnapshotId = contextSnapshotIdByKey.get(consumer.contextKey)
3558
+ if (contextSnapshotId === undefined) throw new Error('parseman: scan consumer lost its context snapshot')
3559
+ const ownSkipPlanId = internOwnSkipPlan(consumer.ownSkip)
3560
+ const ownSkipPlan = ownSkipPlanId === undefined ? undefined : ownSkipPlans[ownSkipPlanId]
3561
+ const status = ownSkipPlan?.entries.find(entry => entry.status.kind !== 'complete')?.status
3562
+ ?? COMPLETE_CAPABILITY
3563
+ const id = scanConsumerPolicies.length
3564
+ const common = {
3565
+ id, siteId, path: `${candidate.path}${consumer.path}`,
3566
+ contextSnapshotId, raw: consumer.raw,
3567
+ ...(ownSkipPlanId === undefined ? {} : { ownSkipPlanId }),
3568
+ lookup: 'dynamic-parse-property-every-skip-test' as const,
3569
+ pendingReuse: 'forbidden' as const,
3570
+ status,
3571
+ }
3572
+ scanConsumerPolicies.push(consumer.kind === 'scanTo' ? {
3573
+ ...common,
3574
+ kind: 'scanTo',
3575
+ ambient: consumer.raw ? 'none-raw' : 'enumerate-trivia-then-scanSkip-per-parse-attempt',
3576
+ context: 'detached-per-attempt-state-errors-only',
3577
+ } : {
3578
+ ...common,
3579
+ kind: 'balanced',
3580
+ ambient: consumer.raw ? 'none-raw' : 'array-identity-interior-cache',
3581
+ cache: 'lookup-every-attempt-enumerate-on-miss',
3582
+ context: 'token-cleared-original',
3583
+ })
3584
+ let ids = scanPolicyIdsBySite.get(siteId)
3585
+ if (ids === undefined) scanPolicyIdsBySite.set(siteId, ids = [])
3586
+ ids.push(id)
3587
+ }
3588
+ }
3589
+
3590
+ const pending = inventory.candidates.map((candidate, id) => {
3591
+ const contextSnapshotId = contextSnapshotIdByKey.get(candidate.contextKey)
3592
+ if (contextSnapshotId === undefined) throw new Error('parseman: capability occurrence lost its context snapshot')
3593
+ const scanConsumerPolicyIds = scanPolicyIdsBySite.get(id) ?? []
3594
+ if (candidate.atom === 'terminal') {
3595
+ const key = keyOf(candidate.parser._def)
3596
+ if (key === undefined) throw new Error('parseman: lexical terminal capability lost its semantic key')
3597
+ const obligations = terminalObligations()
3598
+ return {
3599
+ id, bodyId: candidate.bodyId,
3600
+ path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
3601
+ recognitionContextKey: candidate.recognitionContextKey,
3602
+ context: candidate.context,
3603
+ recognitionContext: candidate.parser._def.tag === 'token'
3604
+ ? { ...candidate.context, trivia: undefined, captureTrivia: false }
3605
+ : candidate.context,
3606
+ semanticKey: key, atom: candidate.atom,
3607
+ parser: candidate.parser, obligations, scanConsumerPolicyIds,
3608
+ status: derivedCapabilityStatus(obligations),
3609
+ }
3610
+ }
3611
+ const def = candidate.parser._def
3612
+ if (candidate.atom === 'choice') {
3613
+ if (def.tag !== 'choice') throw new Error('parseman: lexical choice capability lost its boundary')
3614
+ const obligations = decisionObligations(decisionInventory.recognitionBySite.get(id)
3615
+ ?? gap('choice outcome recognition record is missing'))
3616
+ return {
3617
+ id, bodyId: candidate.bodyId, path: candidate.path, contextSnapshotId,
3618
+ contextKey: candidate.contextKey,
3619
+ recognitionContextKey: candidate.recognitionContextKey,
3620
+ context: candidate.context,
3621
+ recognitionContext: candidate.context,
3622
+ semanticKey: `C\u0000${def.parsers.length}\u0000${def.strategy.tag}`,
3623
+ atom: candidate.atom, parser: candidate.parser, obligations, scanConsumerPolicyIds,
3624
+ status: derivedCapabilityStatus(obligations),
3625
+ }
3626
+ }
3627
+ if (candidate.atom === 'dispatch') {
3628
+ if (def.tag !== 'dispatch') throw new Error('parseman: lexical dispatch capability lost its boundary')
3629
+ const obligations = decisionObligations(decisionInventory.recognitionBySite.get(id)
3630
+ ?? gap('dispatch outcome recognition record is missing'))
3631
+ return {
3632
+ id, bodyId: candidate.bodyId, path: candidate.path, contextSnapshotId,
3633
+ contextKey: candidate.contextKey,
3634
+ recognitionContextKey: candidate.recognitionContextKey,
3635
+ context: candidate.context,
3636
+ recognitionContext: candidate.context,
3637
+ semanticKey: `D\u0000${def.cases.length}\u0000${def.matchers?.length ?? 0}\u0000${def.otherwise === undefined ? 0 : 1}`,
3638
+ atom: candidate.atom, parser: candidate.parser, obligations, scanConsumerPolicyIds,
3639
+ status: derivedCapabilityStatus(obligations),
3640
+ }
3641
+ }
3642
+ if (def.tag !== 'token') throw new Error('parseman: lexical token capability lost its boundary')
3643
+ const session = newNormalizeSession()
3644
+ const normalized = normalizeBalancedLexical(
3645
+ candidate.parser, resolve, session,
3646
+ ) ?? normalizeLexical(candidate.parser, resolve, new Set(), session)
3647
+ if ('refusal' in normalized) {
3648
+ const obligations = tokenObligations(gap(`token normalization: ${normalized.refusal}`))
3649
+ return {
3650
+ id, bodyId: candidate.bodyId,
3651
+ path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
3652
+ recognitionContextKey: candidate.recognitionContextKey,
3653
+ context: candidate.context,
3654
+ recognitionContext: { ...candidate.context, trivia: undefined, captureTrivia: false },
3655
+ semanticKey: `T\u0000GAP\u0000${normalized.refusal}`,
3656
+ atom: candidate.atom, parser: candidate.parser, obligations, scanConsumerPolicyIds,
3657
+ status: derivedCapabilityStatus(obligations),
3658
+ }
3659
+ }
3660
+ const semanticKey = `T\u0000${JSON.stringify(normalized.ir)}`
3661
+ if (matchesEmpty(def.parser)) {
3662
+ const obligations = tokenObligations({
3663
+ kind: 'impossible',
3664
+ proof: 'an atomic source token must consume positive width but this token body is nullable',
3665
+ })
3666
+ return {
3667
+ id, bodyId: candidate.bodyId,
3668
+ path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
3669
+ recognitionContextKey: candidate.recognitionContextKey,
3670
+ context: candidate.context,
3671
+ recognitionContext: { ...candidate.context, trivia: undefined, captureTrivia: false },
3672
+ semanticKey, atom: candidate.atom, parser: candidate.parser, scanConsumerPolicyIds,
3673
+ obligations, status: derivedCapabilityStatus(obligations),
3674
+ }
3675
+ }
3676
+ const diagnosticPlanId = internTransitionDiagnostics(semanticKey, session)
3677
+ const controlPlanId = internControlPlan(session)
3678
+ const topology = internBoundaryTopology(session)
3679
+ const topologyRepresentation = 'id' in topology
3680
+ ? COMPLETE_CAPABILITY : gap(topology.gap)
3681
+ const obligations = directExecutableTokenBody(candidate.parser, resolve) === undefined
3682
+ ? tokenObligations(COMPLETE_CAPABILITY, {
3683
+ diagnostics: true,
3684
+ boundaryPlan: topologyRepresentation,
3685
+ materializationPlan: topologyRepresentation,
3686
+ })
3687
+ : executableLexBodyObligations()
3688
+ return {
3689
+ id, bodyId: candidate.bodyId,
3690
+ path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
3691
+ recognitionContextKey: candidate.recognitionContextKey,
3692
+ context: candidate.context,
3693
+ recognitionContext: { ...candidate.context, trivia: undefined, captureTrivia: false },
3694
+ semanticKey, atom: candidate.atom, parser: candidate.parser, diagnosticPlanId,
3695
+ scanConsumerPolicyIds,
3696
+ controlPlanId,
3697
+ ...('id' in topology ? { boundaryTopologyId: topology.id } : {}),
3698
+ obligations, status: derivedCapabilityStatus(obligations),
3699
+ }
3700
+ })
3701
+ const languageKeys = [...new Set(pending.map(site => `${site.atom}\u0000${site.semanticKey}`))].sort()
3702
+ const languageIdByKey = new Map(languageKeys.map((key, id) => [key, id]))
3703
+ const languages = languageKeys.map((key, id): LexicalCapabilityLanguage => {
3704
+ const split = key.indexOf('\u0000')
3705
+ return {
3706
+ id,
3707
+ atom: key.slice(0, split) as LexicalCapabilitySite['atom'],
3708
+ semanticKey: key.slice(split + 1),
3709
+ }
3710
+ })
3711
+ const preliminaryCapabilities = pending.map(site => ({
3712
+ ...site,
3713
+ languageId: languageIdByKey.get(`${site.atom}\u0000${site.semanticKey}`)!,
3714
+ }))
3715
+ const bindingProjections: LexicalBindingProjection[] = []
3716
+ const bindingEdges = inventory.bindingEdges.map((edge, id): LexicalBindingEdge => {
3717
+ const projection = fixedBindingProjection(edge, id, bindingProjections.length)
3718
+ bindingProjections.push(projection)
3719
+ const {
3720
+ parentBodyId: _parentBodyId, childBodyId: _childBodyId,
3721
+ childOrdinal: _childOrdinal, ...publicEdge
3722
+ } = edge
3723
+ // This record is executable evidence rather than a status assertion: its
3724
+ // numeric reference operand, captured/CSP slot+template, named symbol+template,
3725
+ // and common digest describe all readers before pricing. Scanner/recovery
3726
+ // edges remain GAP until their missing reader templates exist.
3727
+ const complete = projection.readerMask === ALL_BINDING_READERS
3728
+ && projection.variantMask === ALL_BINDING_VARIANTS
3729
+ return {
3730
+ id, ...publicEdge, projectionId: projection.id,
3731
+ contextSnapshotId: contextSnapshotIdByKey.get(edge.contextKey)
3732
+ ?? (() => { throw new Error('parseman: binding edge lost its context snapshot') })(),
3733
+ status: complete ? COMPLETE_CAPABILITY : gap(FIXED_TUPLE_BINDING_GAP),
3734
+ }
3735
+ })
3736
+ const terminalProjections = preliminaryCapabilities
3737
+ .filter(site => site.atom === 'terminal')
3738
+ .map((site, id): LexicalTerminalProjection => {
3739
+ const incomingBindingProjectionIds: number[] = []
3740
+ let readerMask = ALL_BINDING_READERS
3741
+ let variantMask = ALL_BINDING_VARIANTS
3742
+ for (let edgeId = 0; edgeId < inventory.bindingEdges.length; edgeId++) {
3743
+ const edge = inventory.bindingEdges[edgeId]!
3744
+ if (edge.childBodyId !== site.bodyId || edge.childContextKey !== site.contextKey) continue
3745
+ const bindingEdge = bindingEdges[edgeId]!
3746
+ const projection = bindingProjections[bindingEdge.projectionId!]
3747
+ if (projection === undefined) throw new Error('parseman: terminal binding projection disappeared')
3748
+ incomingBindingProjectionIds.push(projection.id)
3749
+ readerMask &= projection.readerMask
3750
+ variantMask &= projection.variantMask
3751
+ }
3752
+ if (incomingBindingProjectionIds.length === 0) {
3753
+ readerMask = 0
3754
+ variantMask = 0
3755
+ }
3756
+ const complete = incomingBindingProjectionIds.length > 0
3757
+ && readerMask === ALL_BINDING_READERS
3758
+ && variantMask === ALL_BINDING_VARIANTS
3759
+ && incomingBindingProjectionIds.every(projectionId =>
3760
+ bindingEdges[bindingProjections[projectionId]!.edgeId]!.status.kind === 'complete')
3761
+ const digestWords = [
3762
+ site.bodyId, site.languageId, site.bodyId,
3763
+ TERMINAL_CAPTURED_TEMPLATE_ID, site.bodyId,
3764
+ readerMask, variantMask, incomingBindingProjectionIds.length,
3765
+ ...incomingBindingProjectionIds.flatMap(projectionId => {
3766
+ const projection = bindingProjections[projectionId]!
3767
+ return [projection.id, projection.semanticDigest]
3768
+ }),
3769
+ ]
3770
+ return {
3771
+ id, siteId: site.id, bodyId: site.bodyId, languageId: site.languageId,
3772
+ referenceBodyId: site.bodyId,
3773
+ capturedTemplateId: TERMINAL_CAPTURED_TEMPLATE_ID,
3774
+ namedSymbolId: site.bodyId,
3775
+ incomingBindingProjectionIds,
3776
+ readerMask, variantMask,
3777
+ semanticDigest: bindingDigest(digestWords, 'terminal', site.semanticKey),
3778
+ status: complete ? COMPLETE_CAPABILITY : gap(FIXED_TUPLE_BINDING_GAP),
3779
+ }
3780
+ })
3781
+ const terminalProjectionBySiteId = new Map(terminalProjections.map(projection =>
3782
+ [projection.siteId, projection]))
3783
+ const terminalCapabilities = preliminaryCapabilities.map((site): LexicalCapabilitySite => {
3784
+ if (site.atom !== 'terminal') return site
3785
+ const projection = terminalProjectionBySiteId.get(site.id)
3786
+ if (projection === undefined) throw new Error('parseman: terminal capability lost its projection')
3787
+ const bindingAndReachability = projection.status.kind === 'complete'
3788
+ ? COMPLETE_PHASE
3789
+ : phaseGap(projection.status, FIXED_TUPLE_BINDING_GAP)
3790
+ const obligations = { ...site.obligations, bindingAndReachability }
3791
+ return { ...site, obligations, status: derivedCapabilityStatus(obligations) }
3792
+ })
3793
+ const decisionEffects = decisionEffectInventory(
3794
+ inventory.candidates, decisionInventory.decisions, bindingEdges, bindingProjections, resolve,
3795
+ )
3796
+ const decisionEffectBySite = new Map(decisionEffects.decisionEffects.map(effect =>
3797
+ [effect.decisionSiteId, effect]))
3798
+ const capabilities = terminalCapabilities.map((site): LexicalCapabilitySite => {
3799
+ if (site.atom !== 'choice' && site.atom !== 'dispatch') return site
3800
+ const effect = decisionEffectBySite.get(site.id)
3801
+ if (effect === undefined) throw new Error('parseman: decision capability lost its effect projection')
3802
+ const obligations = executableDecisionObligations(
3803
+ site.obligations.recognition.representation,
3804
+ effect.phase,
3805
+ )
3806
+ return { ...site, obligations, status: derivedCapabilityStatus(obligations) }
3807
+ })
3808
+ return {
3809
+ capabilities, languages, bindingEdges, bindingProjections, terminalProjections,
3810
+ decisionFamilies: decisionInventory.families,
3811
+ decisionOutcomes: decisionInventory.outcomes,
3812
+ decisions: decisionInventory.decisions,
3813
+ decisionEffectPlan: decisionEffects,
3814
+ transitionDiagnostics,
3815
+ controlPlans,
3816
+ boundaryPlans: boundaryTopologies.length === 0
3817
+ ? [] : [{ id: 0, kind: 'token-context-transaction' }],
3818
+ materializationPlans: boundaryTopologies.length === 0
3819
+ ? [] : [{ id: 0, kind: 'token-source-range' }],
3820
+ grammarWrapperSpecs,
3821
+ grammarCaptureTriviaKinds,
3822
+ boundaryTopologies,
3823
+ contextSnapshots,
3824
+ ownSkipPlans,
3825
+ scanConsumerPolicies,
3826
+ }
3827
+ }
3828
+
3829
+ /** Re-enumerate the final graph so a dropped/filtered candidate fails closed. */
3830
+ export function assertLexicalCapabilityClosure(
3831
+ roots: ReadonlyArray<Combinator<unknown>>,
3832
+ alphabet: Pick<LexicalCapabilityInventory,
3833
+ 'capabilities' | 'capabilityLanguages' | 'bindingEdges' | 'decisionFamilies'
3834
+ | 'decisionOutcomes' | 'decisions'> & Partial<Pick<LexicalCapabilityInventory,
3835
+ 'bindingProjections' | 'terminalProjections' | 'decisionEffectPlan'
3836
+ | 'transitionDiagnostics' | 'boundaryPlans' | 'materializationPlans'
3837
+ | 'grammarWrapperSpecs' | 'grammarCaptureTriviaKinds' | 'boundaryTopologies'
3838
+ | 'controlPlans' | 'contextSnapshots' | 'ownSkipPlans' | 'scanConsumerPolicies'>>,
3839
+ resolve?: (name: string) => Combinator<unknown> | undefined,
3840
+ ): void {
3841
+ const actual = lexicalCapabilityInventory(roots, resolve)
3842
+ const signature = (site: LexicalCapabilitySite): string =>
3843
+ `${site.id}\u0000${site.bodyId}\u0000${site.languageId}\u0000${site.path}\u0000${site.contextKey}\u0000${site.contextSnapshotId}\u0000${site.recognitionContextKey}\u0000${site.atom}\u0000${site.semanticKey}\u0000${site.diagnosticPlanId ?? -1}\u0000${site.controlPlanId ?? -1}\u0000${site.boundaryTopologyId ?? -1}\u0000${site.scanConsumerPolicyIds.join(',')}\u0000${JSON.stringify(site.obligations)}`
3844
+ const expectedKeys = actual.capabilities.map(signature)
3845
+ const suppliedKeys = alphabet.capabilities.map(signature)
3846
+ const expectedLanguages = actual.languages.map(language =>
3847
+ `${language.id}\u0000${language.atom}\u0000${language.semanticKey}`)
3848
+ const suppliedLanguages = alphabet.capabilityLanguages.map(language =>
3849
+ `${language.id}\u0000${language.atom}\u0000${language.semanticKey}`)
3850
+ const edgeSignature = (edge: LexicalBindingEdge): string =>
3851
+ `${edge.id}\u0000${edge.path}\u0000${edge.contextKey}\u0000${edge.childContextKey}\u0000${edge.contextSnapshotId}\u0000${edge.parentTag}\u0000${edge.childTag}\u0000${edge.projectionId ?? -1}\u0000${JSON.stringify(edge.status)}`
3852
+ const expectedEdges = actual.bindingEdges.map(edgeSignature)
3853
+ const suppliedEdges = alphabet.bindingEdges.map(edgeSignature)
3854
+ const expectedBindingProjections = actual.bindingProjections.map(plan => JSON.stringify(plan))
3855
+ const suppliedBindingProjections = (alphabet.bindingProjections ?? []).map(plan => JSON.stringify(plan))
3856
+ const expectedTerminalProjections = actual.terminalProjections.map(plan => JSON.stringify(plan))
3857
+ const suppliedTerminalProjections = (alphabet.terminalProjections ?? []).map(plan => JSON.stringify(plan))
3858
+ const familySignature = (family: LexicalDecisionFamily): string =>
3859
+ `${family.id}\u0000${family.semanticKey}\u0000${JSON.stringify(family.ir)}`
3860
+ const expectedFamilies = actual.decisionFamilies.map(familySignature)
3861
+ const suppliedFamilies = alphabet.decisionFamilies.map(familySignature)
3862
+ const outcomeSignature = (outcome: LexicalDecisionOutcome): string =>
3863
+ `${outcome.id}\u0000${outcome.familyId}\u0000${outcome.semanticKey}\u0000${JSON.stringify(outcome.view)}`
3864
+ const expectedOutcomes = actual.decisionOutcomes.map(outcomeSignature)
3865
+ const suppliedOutcomes = alphabet.decisionOutcomes.map(outcomeSignature)
3866
+ const decisionSignature = (decision: LexicalDecisionSite): string => JSON.stringify(decision)
3867
+ const expectedDecisions = actual.decisions.map(decisionSignature)
3868
+ const suppliedDecisions = alphabet.decisions.map(decisionSignature)
3869
+ const c2Signature = (value: unknown): string => JSON.stringify(value, (_key, entry: unknown) =>
3870
+ typeof entry === 'function' ? String(entry) : entry)
3871
+ const expectedC2 = c2Signature(actual.decisionEffectPlan)
3872
+ const suppliedC2 = c2Signature(alphabet.decisionEffectPlan ?? {})
3873
+ const planSignature = (plan: LexicalTransitionDiagnosticPlan): string => JSON.stringify(plan)
3874
+ const expectedPlans = actual.transitionDiagnostics.map(planSignature)
3875
+ const suppliedPlans = (alphabet.transitionDiagnostics ?? []).map(planSignature)
3876
+ const expectedControlPlans = actual.controlPlans.map(plan => JSON.stringify(plan))
3877
+ const suppliedControlPlans = (alphabet.controlPlans ?? []).map(plan => JSON.stringify(plan))
3878
+ const expectedBoundaryPlans = actual.boundaryPlans.map(plan => JSON.stringify(plan))
3879
+ const suppliedBoundaryPlans = (alphabet.boundaryPlans ?? []).map(plan => JSON.stringify(plan))
3880
+ const expectedMaterializationPlans = actual.materializationPlans.map(plan => JSON.stringify(plan))
3881
+ const suppliedMaterializationPlans = (alphabet.materializationPlans ?? []).map(plan => JSON.stringify(plan))
3882
+ const expectedGrammarSpecs = actual.grammarWrapperSpecs.map(plan => JSON.stringify(plan))
3883
+ const suppliedGrammarSpecs = (alphabet.grammarWrapperSpecs ?? []).map(plan => JSON.stringify(plan))
3884
+ const expectedCaptureKinds = actual.grammarCaptureTriviaKinds.map(plan => JSON.stringify(plan))
3885
+ const suppliedCaptureKinds = (alphabet.grammarCaptureTriviaKinds ?? []).map(plan => JSON.stringify(plan))
3886
+ const expectedTopologies = actual.boundaryTopologies.map(plan => JSON.stringify(plan))
3887
+ const suppliedTopologies = (alphabet.boundaryTopologies ?? []).map(plan => JSON.stringify(plan))
3888
+ const expectedContexts = actual.contextSnapshots.map(plan => JSON.stringify(plan))
3889
+ const suppliedContexts = (alphabet.contextSnapshots ?? []).map(plan => JSON.stringify(plan))
3890
+ const expectedOwnSkip = actual.ownSkipPlans.map(plan => JSON.stringify(plan))
3891
+ const suppliedOwnSkip = (alphabet.ownSkipPlans ?? []).map(plan => JSON.stringify(plan))
3892
+ const expectedScanPolicies = actual.scanConsumerPolicies.map(plan => JSON.stringify(plan))
3893
+ const suppliedScanPolicies = (alphabet.scanConsumerPolicies ?? []).map(plan => JSON.stringify(plan))
3894
+ if (expectedKeys.length !== suppliedKeys.length
3895
+ || expectedKeys.some((key, index) => key !== suppliedKeys[index])
3896
+ || expectedLanguages.length !== suppliedLanguages.length
3897
+ || expectedLanguages.some((key, index) => key !== suppliedLanguages[index])
3898
+ || expectedEdges.length !== suppliedEdges.length
3899
+ || expectedEdges.some((key, index) => key !== suppliedEdges[index])
3900
+ || expectedBindingProjections.length !== suppliedBindingProjections.length
3901
+ || expectedBindingProjections.some((key, index) => key !== suppliedBindingProjections[index])
3902
+ || expectedTerminalProjections.length !== suppliedTerminalProjections.length
3903
+ || expectedTerminalProjections.some((key, index) => key !== suppliedTerminalProjections[index])
3904
+ || expectedFamilies.length !== suppliedFamilies.length
3905
+ || expectedFamilies.some((key, index) => key !== suppliedFamilies[index])
3906
+ || expectedOutcomes.length !== suppliedOutcomes.length
3907
+ || expectedOutcomes.some((key, index) => key !== suppliedOutcomes[index])
3908
+ || expectedDecisions.length !== suppliedDecisions.length
3909
+ || expectedDecisions.some((key, index) => key !== suppliedDecisions[index])
3910
+ || expectedC2 !== suppliedC2
3911
+ || actual.decisionEffectPlan.decisionCallbackAuthorities.length
3912
+ !== (alphabet.decisionEffectPlan?.decisionCallbackAuthorities.length ?? 0)
3913
+ || actual.decisionEffectPlan.decisionCallbackAuthorities.some((entry, index) =>
3914
+ entry.callback !== alphabet.decisionEffectPlan?.decisionCallbackAuthorities[index]?.callback)
3915
+ || expectedPlans.length !== suppliedPlans.length
3916
+ || expectedPlans.some((key, index) => key !== suppliedPlans[index])
3917
+ || expectedControlPlans.length !== suppliedControlPlans.length
3918
+ || expectedControlPlans.some((key, index) => key !== suppliedControlPlans[index])
3919
+ || expectedBoundaryPlans.length !== suppliedBoundaryPlans.length
3920
+ || expectedBoundaryPlans.some((key, index) => key !== suppliedBoundaryPlans[index])
3921
+ || expectedMaterializationPlans.length !== suppliedMaterializationPlans.length
3922
+ || expectedMaterializationPlans.some((key, index) => key !== suppliedMaterializationPlans[index])
3923
+ || expectedGrammarSpecs.length !== suppliedGrammarSpecs.length
3924
+ || expectedGrammarSpecs.some((key, index) => key !== suppliedGrammarSpecs[index])
3925
+ || expectedCaptureKinds.length !== suppliedCaptureKinds.length
3926
+ || expectedCaptureKinds.some((key, index) => key !== suppliedCaptureKinds[index])
3927
+ || expectedTopologies.length !== suppliedTopologies.length
3928
+ || expectedTopologies.some((key, index) => key !== suppliedTopologies[index])
3929
+ || expectedContexts.length !== suppliedContexts.length
3930
+ || expectedContexts.some((key, index) => key !== suppliedContexts[index])
3931
+ || expectedOwnSkip.length !== suppliedOwnSkip.length
3932
+ || expectedOwnSkip.some((key, index) => key !== suppliedOwnSkip[index])
3933
+ || expectedScanPolicies.length !== suppliedScanPolicies.length
3934
+ || expectedScanPolicies.some((key, index) => key !== suppliedScanPolicies[index])) {
3935
+ throw new Error('parseman: lexical capability census is incomplete after final grammar resolution')
3936
+ }
3937
+ }
3938
+
3939
+ /** Phase-A only: inventory obligations without constructing unused runtime families/sites. */
3940
+ export function collectLexicalCapabilities(
3941
+ roots: ReadonlyArray<Combinator<unknown>>,
3942
+ resolve?: (name: string) => Combinator<unknown> | undefined,
3943
+ ): LexicalCapabilityInventory {
3944
+ const inventory = lexicalCapabilityInventory(roots, resolve)
3945
+ const { languages: capabilityLanguages, ...fields } = inventory
3946
+ return {
3947
+ ...fields,
3948
+ capabilityLanguages,
3949
+ capabilityComplete: inventory.capabilities.every(site => site.status.kind !== 'gap')
3950
+ && inventory.bindingEdges.every(edge => edge.status.kind !== 'gap')
3951
+ && inventory.decisionEffectPlan.decisionEffects.every(effect =>
3952
+ effect.phase.executableLowering.kind !== 'gap')
3953
+ && inventory.scanConsumerPolicies.every(policy => policy.status.kind !== 'gap'),
3954
+ }
3955
+ }
3956
+
3957
+ function tokenBoundary(
3958
+ parser: Combinator<unknown>,
3959
+ resolve?: (name: string) => Combinator<unknown> | undefined,
3960
+ seen: Set<Combinator<unknown>> = new Set(),
3961
+ selectorEffects = false,
3962
+ ): { token: Combinator<unknown>; selectorEffects: boolean } | undefined {
3963
+ if (seen.has(parser)) return undefined
3964
+ seen.add(parser)
3965
+ const def = parser._def
3966
+ if (def.tag === 'token') return { token: parser, selectorEffects }
3967
+ if (def.tag === 'lazy') {
3968
+ const target = resolvedLazyTarget(parser, resolve)
3969
+ return target === undefined ? undefined : tokenBoundary(target, resolve, seen, true)
3970
+ }
3971
+ // Value-preserving does NOT mean effect-free: label/field/expect/attempt and
3972
+ // scopes carry diagnostics, capture, commitment, or state. Do not inspect
3973
+ // through them until a consumer proves those effects still execute.
3974
+ return undefined
3975
+ }
3976
+
3977
+ /**
3978
+ * Catalog authored lexical tokens and their dispatch views without confusing
3979
+ * private child terminals for source tokens. This is metadata only: consumers
3980
+ * still have to lower the canonical IR in every shipping engine before enabling
3981
+ * an admission path.
3982
+ */
3983
+ export function collectLexicalAlphabet(
3984
+ roots: ReadonlyArray<Combinator<unknown>>,
3985
+ resolve?: (name: string) => Combinator<unknown> | undefined,
3986
+ ): LexicalAlphabet {
3987
+ const capabilityInventory = collectLexicalCapabilities(roots, resolve)
3988
+ const { capabilities } = capabilityInventory
3989
+ const capabilityFamilyIdByKey = new Map(capabilityInventory.decisionFamilies.map(family =>
3990
+ [family.semanticKey, family.id]))
3991
+ const capabilityTokenFamilies = new Map<Combinator<unknown>, Map<number, LexicalDecisionFamily>>()
3992
+ for (const site of capabilities) {
3993
+ if (site.atom !== 'token' || site.obligations.recognition.representation.kind !== 'complete') continue
3994
+ const semanticKey = site.semanticKey.slice(2)
3995
+ const familyId = capabilityFamilyIdByKey.get(semanticKey)
3996
+ const family = familyId === undefined ? undefined
3997
+ : capabilityInventory.decisionFamilies.find(entry => entry.id === familyId)
3998
+ if (family === undefined) {
3999
+ throw new Error('parseman: completed token occurrence has no canonical capability family')
4000
+ }
4001
+ let byId = capabilityTokenFamilies.get(site.parser)
4002
+ if (byId === undefined) capabilityTokenFamilies.set(site.parser, byId = new Map())
4003
+ byId.set(family.id, family)
4004
+ }
4005
+ const tokenParsers: Combinator<unknown>[] = []
4006
+ const dispatchParsers: Combinator<unknown>[] = []
4007
+ const seenTop = new Set<Combinator<unknown>>()
4008
+ const seenOwned = new Set<Combinator<unknown>>()
4009
+
4010
+ const walk = (parser: Combinator<unknown>, owner?: Combinator<unknown>): void => {
4011
+ const seen = owner === undefined ? seenTop : seenOwned
4012
+ if (seen.has(parser)) return
4013
+ seen.add(parser)
4014
+ if (owner === undefined && parser._def.tag === 'token') tokenParsers.push(parser)
4015
+ if (owner === undefined && parser._def.tag === 'dispatch') dispatchParsers.push(parser)
4016
+ const nextOwner = owner ?? (parser._def.tag === 'token' ? parser : undefined)
4017
+ for (const entry of tokenChildren(parser, resolve)) walk(entry, nextOwner)
4018
+ }
4019
+ for (const root of roots) walk(root)
4020
+
4021
+ const familyIdOf = new Map<Combinator<unknown>, number>()
4022
+ const recognizers: LexicalRecognizer[] = []
4023
+ const diagnostics: LexicalDiagnosticPlan[] = []
4024
+ const recognizerByKey = new Map<string, number>()
4025
+ const sites: LexicalTokenSite[] = []
4026
+ for (const parser of tokenParsers) {
4027
+ const def = parser._def
4028
+ if (def.tag !== 'token') continue
4029
+ const candidateFamilies = capabilityTokenFamilies.get(parser)
4030
+ if (candidateFamilies === undefined) {
4031
+ const refused = normalizeBalancedLexical(parser, resolve)
4032
+ ?? normalizeLexical(def.parser, resolve, new Set())
4033
+ if ('refusal' in refused) {
4034
+ sites.push({ parser, body: def.parser, refusal: refused.refusal })
4035
+ continue
4036
+ }
4037
+ if (matchesEmpty(def.parser)) {
4038
+ sites.push({ parser, body: def.parser, refusal: 'token body may match empty' })
4039
+ continue
4040
+ }
4041
+ throw new Error('parseman: successful legacy recognizer has no canonical capability family')
4042
+ }
4043
+ if (candidateFamilies.size !== 1) {
4044
+ sites.push({
4045
+ parser, body: def.parser,
4046
+ refusal: 'token parser is reused in incompatible lexical contexts',
4047
+ })
4048
+ continue
4049
+ }
4050
+ if (matchesEmpty(def.parser)) {
4051
+ sites.push({ parser, body: def.parser, refusal: 'token body may match empty' })
4052
+ continue
4053
+ }
4054
+ const diagnosticId = diagnostics.length
4055
+ diagnostics.push({ id: diagnosticId, body: def.parser })
4056
+ const canonicalFamily = [...candidateFamilies.values()][0]!
4057
+ const ir = canonicalFamily.ir
4058
+ const key = JSON.stringify(ir)
4059
+ let recognizerId = recognizerByKey.get(key)
4060
+ if (recognizerId === undefined) {
4061
+ recognizerId = recognizers.length
4062
+ recognizerByKey.set(key, recognizerId)
4063
+ recognizers.push({
4064
+ id: recognizerId, key, ir, capabilityFamilyId: canonicalFamily.id,
4065
+ })
4066
+ }
4067
+ const familyId = FIRST_LEXICAL_FAMILY_ID + recognizerId
4068
+ familyIdOf.set(parser, familyId)
4069
+ sites.push({ parser, body: def.parser, diagnosticId, recognizerId, familyId })
4070
+ }
4071
+ const families: LexicalTokenFamily[] = recognizers.map(recognizer => ({
4072
+ id: FIRST_LEXICAL_FAMILY_ID + recognizer.id,
4073
+ recognizerId: recognizer.id,
4074
+ }))
4075
+
4076
+ let nextOutcomeId = FIRST_LEXICAL_FAMILY_ID + families.length
4077
+ const outcomes: LexicalOutcomeSpec[] = []
4078
+ const outcomeIdByKey = new Map<string, number>()
4079
+ const internOutcome = (familyId: number, match: LexicalOutcomeMatch): number => {
4080
+ if (match.kind === 'exact' && match.values.length !== 1) {
4081
+ throw new Error('parseman: global exact lexical outcomes must be atomic')
4082
+ }
4083
+ const key = `${familyId}\u0000${canonicalLexicalOutcomeKey(match)}`
4084
+ const prior = outcomeIdByKey.get(key)
4085
+ if (prior !== undefined) return prior
4086
+ const id = nextOutcomeId++
4087
+ outcomeIdByKey.set(key, id)
4088
+ outcomes.push({ id, familyId, match })
4089
+ return id
4090
+ }
4091
+ const classifiers: LexicalTokenClassifier[] = []
4092
+ for (const parser of dispatchParsers) {
4093
+ const def = parser._def
4094
+ if (def.tag !== 'dispatch') continue
4095
+ const boundary = tokenBoundary(def.selector, resolve)
4096
+ const familyId = boundary === undefined ? undefined : familyIdOf.get(boundary.token)
4097
+ if (familyId === undefined) continue
4098
+ const outcomes: LexicalTokenOutcome[] = []
4099
+ const routes: LexicalTokenRoute[] = []
4100
+ const exclusions: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[] = []
4101
+ for (const entry of def.cases) {
4102
+ // Exact subtype ids are ATOMIC even when one dispatch arm groups keys.
4103
+ // The site route is represented by multiple outcomes pointing at the same
4104
+ // parser, so URL_OPEN keeps one global id whether another site groups it
4105
+ // with VAR_OPEN/CALC_OPEN or routes it alone.
4106
+ const matches: LexicalOutcomeMatch[] = []
4107
+ const acceptedIds: number[] = []
4108
+ for (const key of entry.keys) {
4109
+ const match: LexicalOutcomeMatch = {
4110
+ kind: 'exact', values: [key], caseInsensitive: entry.caseInsensitive,
4111
+ }
4112
+ const id = internOutcome(familyId, match)
4113
+ exclusions.push(match)
4114
+ matches.push(match)
4115
+ acceptedIds.push(id)
4116
+ outcomes.push({ id, match })
4117
+ }
4118
+ routes.push({
4119
+ index: routes.length, kind: 'exact', acceptedIds, matches,
4120
+ parser: entry.parser, usesRouted: branchUsesRouted(entry),
4121
+ })
4122
+ }
4123
+ for (const entry of def.matchers ?? []) {
4124
+ const match: LexicalOutcomeMatch = entry.kind === 'matches'
4125
+ ? { kind: 'matches', value: entry.value, flags: entry.flags ?? '', caseInsensitive: entry.caseInsensitive }
4126
+ : { kind: entry.kind, value: entry.value, caseInsensitive: entry.caseInsensitive }
4127
+ exclusions.push(match)
4128
+ const id = internOutcome(familyId, match)
4129
+ outcomes.push({ id, match })
4130
+ routes.push({
4131
+ index: routes.length, kind: 'matcher', acceptedIds: [id], matches: [match],
4132
+ parser: entry.parser, usesRouted: branchUsesRouted(entry),
4133
+ })
4134
+ }
4135
+ if (def.otherwise !== undefined) {
4136
+ const match: LexicalOutcomeMatch = { kind: 'otherwise', excluding: exclusions }
4137
+ const id = internOutcome(familyId, match)
4138
+ outcomes.push({ id, match })
4139
+ routes.push({
4140
+ index: routes.length, kind: 'otherwise', acceptedIds: [id], matches: [match],
4141
+ parser: def.otherwise,
4142
+ usesRouted: branchUsesRouted({ parser: def.otherwise, usesRouted: def.otherwiseUsesRouted }),
4143
+ })
4144
+ }
4145
+ for (const route of routes) {
4146
+ if (route.matches.length !== route.acceptedIds.length) {
4147
+ throw new Error('parseman: lexical route match/id arity mismatch')
4148
+ }
4149
+ }
4150
+ classifiers.push({
4151
+ dispatch: parser, familyId, selectorEffects: boundary?.selectorEffects ?? false, outcomes, routes,
4152
+ })
4153
+ }
4154
+
4155
+ /*
4156
+ * IDs ARE CONTENT-ORDERED, not discovery-ordered. The collector walks the
4157
+ * final winner graph once, but compose(), a rule-map literal, and the macro
4158
+ * evaluator are allowed to present equal roots in different insertion order.
4159
+ * Sorting the already-canonical recognizer keys and family-qualified outcome
4160
+ * keys makes those routes agree without publishing a hash or an authored
4161
+ * token-instance identity in the artifact.
4162
+ */
4163
+ const recognizerOrder = [...recognizers].sort((a, b) =>
4164
+ a.key < b.key ? -1 : a.key > b.key ? 1 : 0)
4165
+ const recognizerId = new Map(recognizerOrder.map((entry, index) => [entry.id, index]))
4166
+ const stableRecognizers = recognizerOrder.map((entry, id) => ({ ...entry, id }))
4167
+ const stableFamilies = stableRecognizers.map(entry => ({
4168
+ id: FIRST_LEXICAL_FAMILY_ID + entry.id,
4169
+ recognizerId: entry.id,
4170
+ }))
4171
+ const stableFamilyId = (oldFamilyId: number): number => {
4172
+ const oldRecognizerId = oldFamilyId - FIRST_LEXICAL_FAMILY_ID
4173
+ const id = recognizerId.get(oldRecognizerId)
4174
+ if (id === undefined) throw new Error('parseman: lexical family has no recognizer')
4175
+ return FIRST_LEXICAL_FAMILY_ID + id
4176
+ }
4177
+ const stableFamilyIdOf = new Map<Combinator<unknown>, number>()
4178
+ for (const [parser, familyId] of familyIdOf) stableFamilyIdOf.set(parser, stableFamilyId(familyId))
4179
+ const stableSites = sites.map(site => site.familyId === undefined ? site : ({
4180
+ ...site,
4181
+ familyId: stableFamilyId(site.familyId),
4182
+ recognizerId: recognizerId.get(site.recognizerId!)!,
4183
+ }))
4184
+
4185
+ const orderedOutcomes = [...outcomes].sort((a, b) => {
4186
+ const ak = `${stableFamilyId(a.familyId)}\u0000${canonicalLexicalOutcomeKey(a.match)}`
4187
+ const bk = `${stableFamilyId(b.familyId)}\u0000${canonicalLexicalOutcomeKey(b.match)}`
4188
+ return ak < bk ? -1 : ak > bk ? 1 : 0
4189
+ })
4190
+ const oldOutcomeToStable = new Map<number, number>()
4191
+ const firstOutcomeId = FIRST_LEXICAL_FAMILY_ID + stableFamilies.length
4192
+ const stableOutcomes = orderedOutcomes.map((entry, index) => {
4193
+ const id = firstOutcomeId + index
4194
+ oldOutcomeToStable.set(entry.id, id)
4195
+ return { ...entry, id, familyId: stableFamilyId(entry.familyId) }
4196
+ })
4197
+ const stableClassifiers = classifiers.map(classifier => ({
4198
+ ...classifier,
4199
+ familyId: stableFamilyId(classifier.familyId),
4200
+ outcomes: classifier.outcomes.map(outcome => ({
4201
+ ...outcome,
4202
+ id: oldOutcomeToStable.get(outcome.id)!,
4203
+ })),
4204
+ routes: classifier.routes.map(route => ({
4205
+ ...route,
4206
+ acceptedIds: route.acceptedIds.map(id => oldOutcomeToStable.get(id)!),
4207
+ })),
4208
+ }))
4209
+
4210
+ // Deliberately do not return `primitiveKernels`: collectAlphabet() remains a
4211
+ // separate historical/kernel API, while this is the only family/site seam.
4212
+ // A choice-admission consumer therefore cannot accidentally publish a child
4213
+ // literal/regex id by reading a field from LexicalAlphabet.
4214
+ return {
4215
+ ...capabilityInventory,
4216
+ recognizers: stableRecognizers,
4217
+ diagnostics,
4218
+ families: stableFamilies,
4219
+ sites: stableSites,
4220
+ outcomes: stableOutcomes,
4221
+ classifiers: stableClassifiers,
4222
+ familyIdOf: stableFamilyIdOf,
4223
+ }
4224
+ }
4225
+
4226
+ function serializeLexicalIr(ir: LexicalIr, out: number[], constant: (value: unknown) => number): void {
4227
+ const start = out.length
4228
+ const begin = (kind: number): void => { out.push(kind, 0) }
4229
+ switch (ir.kind) {
4230
+ case 'literal':
4231
+ begin(0); out.push(constant(ir.value), ir.caseInsensitive ? 1 : 0)
4232
+ break
4233
+ case 'keywords':
4234
+ begin(1)
4235
+ out.push(ir.caseInsensitive ? 1 : 0, ir.boundary === undefined ? -1 : constant(ir.boundary), ir.words.length)
4236
+ for (const word of ir.words) out.push(constant(word))
4237
+ break
4238
+ case 'regex':
4239
+ begin(2); out.push(constant(new RegExp(ir.source, ir.flags)))
4240
+ break
4241
+ case 'sequence':
4242
+ begin(3); out.push(ir.parts.length)
4243
+ for (const part of ir.parts) serializeLexicalIr(part, out, constant)
4244
+ break
4245
+ case 'choice':
4246
+ begin(4); out.push(ir.arms.length)
4247
+ for (const arm of ir.arms) serializeLexicalIr(arm, out, constant)
4248
+ break
4249
+ case 'repeat':
4250
+ begin(5)
4251
+ out.push(ir.min, ir.max ?? -1, ir.greedy ? 1 : 0, ir.mode === 'possessive' ? 0 : 1)
4252
+ serializeLexicalIr(ir.body, out, constant)
4253
+ break
4254
+ case 'assert':
4255
+ begin(6); out.push(ir.positive ? 1 : 0)
4256
+ serializeLexicalIr(ir.body, out, constant)
4257
+ break
4258
+ case 'balanced':
4259
+ begin(7)
4260
+ out.push(constant(ir.open), constant(ir.close), ir.strict ? 1 : 0, ir.raw ? 1 : 0, ir.skip.length)
4261
+ for (const entry of ir.skip) serializeLexicalIr(entry, out, constant)
4262
+ break
4263
+ }
4264
+ out[start + 1] = out.length - start
4265
+ }
4266
+
4267
+ /**
4268
+ * Project the compiler graph into compact numeric pools. The caller supplies
4269
+ * only already-relocated table-site numbers and its existing const-pool intern;
4270
+ * no combinator or identity map crosses this boundary.
4271
+ */
4272
+ export function serializeLexicalPlan(
4273
+ alphabet: LexicalAlphabet,
4274
+ constant: (value: unknown) => number,
4275
+ tokenSites: readonly number[],
4276
+ dispatchSites: ReadonlyArray<{ readonly dsp: number; readonly classifier: LexicalTokenClassifier }>,
4277
+ ): NumericLexicalPlan | undefined {
4278
+ // Phase B is globally disabled until phase A has a total replacement body
4279
+ // for every reachable token-capable atom in every supported variant. A
4280
+ // partially supported graph is never serialized as a cheaper token plan.
4281
+ if (!alphabet.capabilityComplete) return undefined
4282
+ if (tokenSites.length === 0) return undefined
4283
+ const recognizerOffsets: number[] = []
4284
+ const recognizerData: number[] = []
4285
+ for (const recognizer of alphabet.recognizers) {
4286
+ if (recognizer.id !== recognizerOffsets.length) throw new Error('parseman: lexical recognizer ids are not dense')
4287
+ recognizerOffsets.push(recognizerData.length)
4288
+ serializeLexicalIr(recognizer.ir, recognizerData, constant)
4289
+ }
4290
+
4291
+ const outcomeOffsets: number[] = []
4292
+ const outcomeData: number[] = []
4293
+ for (const outcome of alphabet.outcomes) {
4294
+ outcomeOffsets.push(outcomeData.length)
4295
+ const match = outcome.match
4296
+ switch (match.kind) {
4297
+ case 'exact':
4298
+ if (match.values.length !== 1) throw new Error('parseman: lexical exact outcome is not atomic')
4299
+ outcomeData.push(outcome.id, outcome.familyId, 0, constant(match.values[0]!), match.caseInsensitive ? 1 : 0)
4300
+ break
4301
+ case 'startsWith':
4302
+ outcomeData.push(outcome.id, outcome.familyId, 1, constant(match.value), match.caseInsensitive ? 1 : 0)
4303
+ break
4304
+ case 'endsWith':
4305
+ outcomeData.push(outcome.id, outcome.familyId, 2, constant(match.value), match.caseInsensitive ? 1 : 0)
4306
+ break
4307
+ case 'matches': {
4308
+ const rawFlags = match.caseInsensitive && !match.flags.includes('i') ? `${match.flags}i` : match.flags
4309
+ const flags = new RegExp('', rawFlags.replace(/g/g, '')).flags
4310
+ outcomeData.push(outcome.id, outcome.familyId, 3, constant(new RegExp(match.value, flags)))
4311
+ break
4312
+ }
4313
+ case 'otherwise':
4314
+ outcomeData.push(outcome.id, outcome.familyId, 4)
4315
+ break
4316
+ }
4317
+ }
4318
+
4319
+ const sites: number[] = []
4320
+ const routes: number[] = []
4321
+ const accepted: number[] = []
4322
+ for (const { dsp, classifier } of [...dispatchSites].sort((a, b) => a.dsp - b.dsp)) {
4323
+ const routeOffset = routes.length
4324
+ for (const route of classifier.routes) {
4325
+ const acceptedOffset = accepted.length
4326
+ accepted.push(...route.acceptedIds)
4327
+ const kind = route.kind === 'exact' ? 0 : route.kind === 'matcher' ? 1 : 2
4328
+ routes.push(route.index, kind | (route.usesRouted ? 4 : 0), acceptedOffset, route.acceptedIds.length)
4329
+ }
4330
+ sites.push(dsp, classifier.familyId, routeOffset, classifier.routes.length)
4331
+ }
4332
+ for (let i = 0; i < tokenSites.length; i += 2) {
4333
+ if (tokenSites[i + 1]! < FIRST_LEXICAL_FAMILY_ID) {
4334
+ throw new Error('parseman: lexical token site used a primitive-terminal id')
4335
+ }
4336
+ }
4337
+ return {
4338
+ recognizerOffsets, recognizerData, outcomeOffsets, outcomeData,
4339
+ tokenSites: [...tokenSites], sites, routes, accepted,
4340
+ }
4341
+ }
4342
+
4343
+ function foldAsciiCode(code: number): number {
4344
+ return code >= 65 && code <= 90 ? code + 32 : code
4345
+ }
4346
+
4347
+ function rangeEquals(input: string, start: number, end: number, value: string, folded: boolean): boolean {
4348
+ if (end - start !== value.length) return false
4349
+ for (let i = 0; i < value.length; i++) {
4350
+ const actual = input.charCodeAt(start + i)
4351
+ const expected = value.charCodeAt(i)
4352
+ if ((folded ? foldAsciiCode(actual) : actual) !== (folded ? foldAsciiCode(expected) : expected)) return false
4353
+ }
4354
+ return true
4355
+ }
4356
+
4357
+ function rangeStartsWith(input: string, start: number, end: number, value: string, folded: boolean): boolean {
4358
+ return start + value.length <= end && rangeEquals(input, start, start + value.length, value, folded)
4359
+ }
4360
+
4361
+ function rangeEndsWith(input: string, start: number, end: number, value: string, folded: boolean): boolean {
4362
+ return end - value.length >= start && rangeEquals(input, end - value.length, end, value, folded)
4363
+ }
4364
+
4365
+ function lexicalOutcomeMatches(
4366
+ match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>,
4367
+ input: string,
4368
+ start: number,
4369
+ end: number,
4370
+ ): boolean {
4371
+ if (match.kind === 'exact') {
4372
+ return match.values.some(value => rangeEquals(input, start, end, value, match.caseInsensitive))
4373
+ }
4374
+ if (match.kind === 'startsWith') return rangeStartsWith(input, start, end, match.value, match.caseInsensitive)
4375
+ if (match.kind === 'endsWith') return rangeEndsWith(input, start, end, match.value, match.caseInsensitive)
4376
+ if ('flags' in match) {
4377
+ const flags = match.caseInsensitive && !match.flags.includes('i') ? `${match.flags}i` : match.flags
4378
+ return new RegExp(match.value, flags).test(input.slice(start, end))
4379
+ }
4380
+ return false
4381
+ }
4382
+
4383
+ /**
4384
+ * Metadata/reference oracle for compatible same-range views. Exact/prefix/suffix
4385
+ * checks read char codes directly; regex matcher views slice only in this cold
4386
+ * oracle. Production token cursors must lower matcher plans without treating
4387
+ * this helper as their allocation contract.
4388
+ */
4389
+ export function compatibleLexicalOutcomes(
4390
+ classifier: LexicalTokenClassifier,
4391
+ input: string,
4392
+ start: number,
4393
+ end: number,
4394
+ ): number[] {
4395
+ const ids: number[] = []
4396
+ let otherwise: number | undefined
4397
+ for (const outcome of classifier.outcomes) {
4398
+ const match = outcome.match
4399
+ if (match.kind === 'otherwise') { otherwise = outcome.id; continue }
4400
+ if (lexicalOutcomeMatches(match, input, start, end) && !ids.includes(outcome.id)) ids.push(outcome.id)
4401
+ }
4402
+ if (ids.length === 0 && otherwise !== undefined) ids.push(otherwise)
4403
+ return ids
4404
+ }
4405
+
4406
+ /**
4407
+ * Cold oracle for dispatch selection precedence. Compatible views are a set;
4408
+ * selected routing is still exact cases first, then matcher source order, then
4409
+ * otherwise, and a selected branch failure remains committed.
4410
+ */
4411
+ export function selectedLexicalOutcome(
4412
+ classifier: LexicalTokenClassifier,
4413
+ input: string,
4414
+ start: number,
4415
+ end: number,
4416
+ ): { route: LexicalTokenRoute; outcomeId: number } | undefined {
4417
+ for (const route of classifier.routes) {
4418
+ for (let i = 0; i < route.matches.length; i++) {
4419
+ const match = route.matches[i]!
4420
+ if (match.kind === 'otherwise' || lexicalOutcomeMatches(match, input, start, end)) {
4421
+ return { route, outcomeId: route.acceptedIds[i]! }
4422
+ }
4423
+ }
4424
+ }
4425
+ return undefined
4426
+ }
4427
+
124
4428
  /** The id already assigned to this terminal, if it is one. */
125
4429
  export function terminalId(alphabet: Alphabet, p: Combinator<unknown>): number | undefined {
126
4430
  const key = keyOf(p._def)