parseman 0.47.1 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -21
- package/dist/analysis/diagnostics.js +2863 -353
- package/dist/analysis/diagnostics.js.map +3 -3
- package/dist/cli/index.js +2854 -344
- package/dist/cli/index.js.map +3 -3
- package/dist/combinators/expect.d.ts +1 -0
- package/dist/combinators/expect.d.ts.map +1 -1
- package/dist/combinators/expected.d.ts +8 -0
- package/dist/combinators/expected.d.ts.map +1 -0
- package/dist/combinators/first-set.d.ts +8 -0
- package/dist/combinators/first-set.d.ts.map +1 -1
- package/dist/combinators/grammar.d.ts.map +1 -1
- package/dist/combinators/keywords.d.ts +4 -0
- package/dist/combinators/keywords.d.ts.map +1 -1
- package/dist/combinators/literal.d.ts.map +1 -1
- package/dist/combinators/not.d.ts.map +1 -1
- package/dist/combinators/peek.d.ts.map +1 -1
- package/dist/combinators/regex.d.ts.map +1 -1
- package/dist/combinators/repeat.d.ts.map +1 -1
- package/dist/combinators/sequence.d.ts.map +1 -1
- package/dist/combinators/trivia-skip.d.ts +16 -3
- package/dist/combinators/trivia-skip.d.ts.map +1 -1
- package/dist/compiler/direct-projection.d.ts +12 -0
- package/dist/compiler/direct-projection.d.ts.map +1 -0
- package/dist/compiler/token-alphabet.d.ts +845 -5
- package/dist/compiler/token-alphabet.d.ts.map +1 -1
- package/dist/compiler/token-capability.d.ts +3 -0
- package/dist/compiler/token-capability.d.ts.map +1 -0
- package/dist/compiler/token-capability.js +3192 -0
- package/dist/compiler/token-capability.js.map +6 -0
- package/dist/cst/trivia-charscan.d.ts +3 -0
- package/dist/cst/trivia-charscan.d.ts.map +1 -1
- package/dist/cst/trivia-css-scanner.d.ts +5 -0
- package/dist/cst/trivia-css-scanner.d.ts.map +1 -0
- package/dist/cst/trivia-kinds.d.ts.map +1 -1
- package/dist/index.js +2862 -339
- package/dist/index.js.map +3 -3
- package/dist/language-service/index.js +5 -1
- package/dist/language-service/index.js.map +1 -1
- package/dist/plugin/index.js +2870 -347
- package/dist/plugin/index.js.map +3 -3
- package/dist/spec/index.js +315 -46
- package/dist/spec/index.js.map +2 -2
- package/dist/table/assemble.d.ts.map +1 -1
- package/dist/table/child-slots.d.ts.map +1 -1
- package/dist/table/compile-rule-map.d.ts +3 -1
- package/dist/table/compile-rule-map.d.ts.map +1 -1
- package/dist/table/emit-assembly.d.ts +12 -13
- package/dist/table/emit-assembly.d.ts.map +1 -1
- package/dist/table/emit.d.ts.map +1 -1
- package/dist/table/encode.d.ts.map +1 -1
- package/dist/table/exec-baseline.d.ts.map +1 -1
- package/dist/table/exec.d.ts.map +1 -1
- package/dist/table/index.js +2819 -300
- package/dist/table/index.js.map +3 -3
- package/dist/table/lex-body.d.ts +9 -0
- package/dist/table/lex-body.d.ts.map +1 -0
- package/dist/table/lex-program.d.ts +25 -0
- package/dist/table/lex-program.d.ts.map +1 -0
- package/dist/table/ops.d.ts +43 -7
- package/dist/table/ops.d.ts.map +1 -1
- package/dist/table/program.d.ts +50 -2
- package/dist/table/program.d.ts.map +1 -1
- package/dist/table/scalar-terminal.d.ts +13 -0
- package/dist/table/scalar-terminal.d.ts.map +1 -0
- package/dist/table/site-labels.d.ts.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/package.json +11 -11
- package/src/combinators/expect.ts +1 -0
- package/src/combinators/expected.ts +17 -0
- package/src/combinators/first-set.ts +58 -5
- package/src/combinators/grammar.ts +8 -3
- package/src/combinators/keywords.ts +22 -6
- package/src/combinators/literal.ts +2 -1
- package/src/combinators/not.ts +5 -1
- package/src/combinators/peek.ts +5 -1
- package/src/combinators/regex.ts +5 -3
- package/src/combinators/repeat.ts +4 -10
- package/src/combinators/sequence.ts +17 -14
- package/src/combinators/trivia-skip.ts +46 -13
- package/src/compiler/direct-projection.ts +43 -0
- package/src/compiler/token-alphabet.ts +4316 -12
- package/src/compiler/token-capability.ts +6 -0
- package/src/cst/trivia-charscan.ts +34 -0
- package/src/cst/trivia-css-scanner.ts +142 -0
- package/src/cst/trivia-kinds.ts +3 -1
- package/src/plugin/index.ts +4 -4
- package/src/table/assemble.ts +1211 -221
- package/src/table/child-slots.ts +2 -1
- package/src/table/compile-rule-map.ts +21 -3
- package/src/table/emit-assembly.ts +328 -85
- package/src/table/emit.ts +16 -2
- package/src/table/encode.ts +242 -50
- package/src/table/exec-baseline.ts +3 -7
- package/src/table/exec.ts +121 -7
- package/src/table/lex-body.ts +30 -0
- package/src/table/lex-program.ts +422 -0
- package/src/table/ops.ts +45 -7
- package/src/table/program.ts +46 -2
- package/src/table/scalar-terminal.ts +51 -0
- package/src/table/site-labels.ts +11 -1
- package/src/table/stamp.ts +1 -1
- package/src/types.ts +1 -1
- package/src/version.ts +1 -1
- package/dist/analysis/diagnostics.cjs +0 -13824
- package/dist/analysis/diagnostics.cjs.map +0 -6
- package/dist/cli/index.cjs +0 -14925
- package/dist/cli/index.cjs.map +0 -6
- package/dist/index.cjs +0 -16391
- package/dist/index.cjs.map +0 -6
- package/dist/language-service/index.cjs +0 -1304
- package/dist/language-service/index.cjs.map +0 -6
- package/dist/oracle/index.cjs +0 -254
- package/dist/oracle/index.cjs.map +0 -6
- package/dist/plugin/index.cjs +0 -16097
- package/dist/plugin/index.cjs.map +0 -6
- package/dist/run/index.cjs +0 -577
- package/dist/run/index.cjs.map +0 -6
- package/dist/spec/index.cjs +0 -6236
- package/dist/spec/index.cjs.map +0 -6
- package/dist/table/index.cjs +0 -11463
- package/dist/table/index.cjs.map +0 -6
|
@@ -1,9 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Derived tokenization
|
|
2
|
+
* Derived tokenization groundwork: primitive kernels and lexical-token families.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* `literal`, `keywords`, and `regex
|
|
6
|
-
*
|
|
4
|
+
* `collectAlphabet()` is the preserved primitive-terminal scanner groundwork:
|
|
5
|
+
* it knows every `literal`, `keywords`, and `regex` in the composed grammar and
|
|
6
|
+
* assigns each one a small integer KERNEL id. Those ids are implementation
|
|
7
|
+
* details, not parser tokens. In particular, unwrapping
|
|
8
|
+
* `token(sequence(identifier, optional(literal('('))))` must never publish an
|
|
9
|
+
* identifier id followed by a `(` id: `token()` defines ONE contiguous source
|
|
10
|
+
* token and ONE CST leaf for the complete range.
|
|
11
|
+
*
|
|
12
|
+
* `collectLexicalAlphabet()` is the production-facing boundary. It assigns ids
|
|
13
|
+
* to authored `token()` families, normalizes effect-free bodies into canonical
|
|
14
|
+
* lexical IR, interns equivalent recognizers, and records the downstream
|
|
15
|
+
* `dispatch()` outcome classifiers over the same range. Primitive terminals
|
|
16
|
+
* stay private recognizer machinery.
|
|
7
17
|
*
|
|
8
18
|
* Two things are deliberately separated, because conflating them is what makes
|
|
9
19
|
* a naive derived scanner useless:
|
|
@@ -20,7 +30,13 @@
|
|
|
20
30
|
* Nothing here decides HOW a token is recognized (see `token-scanner.ts`); this
|
|
21
31
|
* module only decides WHAT the tokens are and WHICH ones each site may see.
|
|
22
32
|
*/
|
|
23
|
-
import type { Combinator, ParserDef } from '../types.ts'
|
|
33
|
+
import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts'
|
|
34
|
+
import { classifyFinalChoice, firstSetOf, intersects, matchesEmpty } from '../combinators/first-set.ts'
|
|
35
|
+
import { branchUsesRouted } from '../combinators/dispatch.ts'
|
|
36
|
+
import type { BalancedSpec } from '../combinators/scanTo.ts'
|
|
37
|
+
import { deriveExpected } from '../combinators/expect.ts'
|
|
38
|
+
import { assertionFailureExpected, directTerminalFailureExpected } from '../combinators/expected.ts'
|
|
39
|
+
import { keywordsRegExp } from '../combinators/keywords.ts'
|
|
24
40
|
|
|
25
41
|
/** A terminal in the derived alphabet, with its globally assigned id. */
|
|
26
42
|
export type TokenTerminal =
|
|
@@ -42,6 +58,1202 @@ export type Alphabet = {
|
|
|
42
58
|
originOf: Map<number, Combinator<unknown>>
|
|
43
59
|
}
|
|
44
60
|
|
|
61
|
+
/** Effect-free recognition IR underneath one authored `token()` boundary. */
|
|
62
|
+
export type LexicalIr =
|
|
63
|
+
| { readonly kind: 'literal'; readonly value: string; readonly caseInsensitive: boolean }
|
|
64
|
+
| { readonly kind: 'keywords'; readonly words: readonly string[]; readonly caseInsensitive: boolean; readonly boundary: string | undefined }
|
|
65
|
+
| { readonly kind: 'regex'; readonly source: string; readonly flags: string }
|
|
66
|
+
| { readonly kind: 'sequence'; readonly parts: readonly LexicalIr[] }
|
|
67
|
+
| { readonly kind: 'choice'; readonly arms: readonly LexicalIr[] }
|
|
68
|
+
| { readonly kind: 'repeat'; readonly body: LexicalIr; readonly min: number; readonly max: number | null; readonly greedy: boolean; readonly mode: 'possessive' | 'backtracking' }
|
|
69
|
+
| { readonly kind: 'assert'; readonly positive: boolean; readonly body: LexicalIr }
|
|
70
|
+
| {
|
|
71
|
+
readonly kind: 'balanced'
|
|
72
|
+
readonly open: string
|
|
73
|
+
readonly close: string
|
|
74
|
+
readonly strict: boolean
|
|
75
|
+
readonly raw: boolean
|
|
76
|
+
/** Ordered explicit skippers; ambient trivia/scanSkip precede them unless raw. */
|
|
77
|
+
readonly skip: readonly LexicalIr[]
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** First complete executable lexical body. It is intentionally a semantic
|
|
81
|
+
* shape rather than an opcode: the encoder relocates the regex into the table
|
|
82
|
+
* pool only after TOKEN wins the compiler-only cost comparison. */
|
|
83
|
+
export type OptionalSuffixLexBody = {
|
|
84
|
+
readonly kind: 'regex-optional-code-unit'
|
|
85
|
+
readonly source: string
|
|
86
|
+
readonly flags: string
|
|
87
|
+
readonly suffix: string
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export type ExecutableOptionalSuffixLexBody = OptionalSuffixLexBody & {
|
|
91
|
+
/** Match the two authored terminal line-publication points exactly. */
|
|
92
|
+
readonly baseCanMatchNewline: boolean
|
|
93
|
+
readonly suffixCanMatchNewline: boolean
|
|
94
|
+
/** Failure authority belongs to the resolved final regex, not the lazy
|
|
95
|
+
* construction thunk that originally named it. */
|
|
96
|
+
readonly expected: readonly string[]
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** A direct authored terminal under token(). The boundary shell still owns
|
|
100
|
+
* source slicing/CST; this body owns exactly the terminal recognizer and lines. */
|
|
101
|
+
export type ExecutableTerminalLexBody = {
|
|
102
|
+
readonly kind: 'regex-terminal'
|
|
103
|
+
readonly source: string
|
|
104
|
+
readonly flags: string
|
|
105
|
+
readonly canMatchNewline: boolean
|
|
106
|
+
readonly expected: readonly string[]
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export type ExecutableOrdered4LexBody = {
|
|
110
|
+
readonly kind: 'ordered4-terminals'
|
|
111
|
+
readonly commonFirstSet: FirstSet
|
|
112
|
+
readonly commonMissExpected: readonly string[]
|
|
113
|
+
readonly expected: readonly string[]
|
|
114
|
+
readonly arms: readonly [
|
|
115
|
+
ExecutableTerminalLexBody,
|
|
116
|
+
ExecutableTerminalLexBody,
|
|
117
|
+
ExecutableTerminalLexBody,
|
|
118
|
+
ExecutableTerminalLexBody,
|
|
119
|
+
]
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export type ExecutableNot2LexBody = {
|
|
123
|
+
readonly kind: 'not2-ordered4-terminal'
|
|
124
|
+
readonly guard0Expected: readonly string[]
|
|
125
|
+
readonly guard1Expected: readonly string[]
|
|
126
|
+
readonly terminalExpected: readonly string[]
|
|
127
|
+
readonly guard0: ExecutableOrdered4LexBody['arms']
|
|
128
|
+
readonly guard1: ExecutableTerminalLexBody
|
|
129
|
+
readonly terminal: ExecutableTerminalLexBody
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export type ExecutableLexNode =
|
|
133
|
+
| { readonly kind: 'terminal'; readonly terminal: ExecutableTerminalLexBody }
|
|
134
|
+
| { readonly kind: 'sequence'; readonly parts: readonly ExecutableLexNode[] }
|
|
135
|
+
| {
|
|
136
|
+
readonly kind: 'choice'
|
|
137
|
+
readonly arms: readonly ExecutableLexNode[]
|
|
138
|
+
readonly firstSets: readonly FirstSet[]
|
|
139
|
+
readonly armExpected: readonly (readonly string[])[]
|
|
140
|
+
readonly expected: readonly string[]
|
|
141
|
+
}
|
|
142
|
+
| { readonly kind: 'not'; readonly body: ExecutableLexNode; readonly expected: readonly string[] }
|
|
143
|
+
| { readonly kind: 'optional'; readonly body: ExecutableLexNode }
|
|
144
|
+
|
|
145
|
+
export type ExecutableFixedLexBody = {
|
|
146
|
+
readonly kind: 'fixed-tree'
|
|
147
|
+
readonly root: ExecutableLexNode
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** A balanced token whose recognition remains owned by the canonical scanner
|
|
151
|
+
* pool. The selected shell removes only the losing token/parent topology; it
|
|
152
|
+
* never restates ambient scanSkip, recovery, or skipper semantics. */
|
|
153
|
+
export type ExecutableBalancedLexBody = {
|
|
154
|
+
readonly kind: 'balanced-scan'
|
|
155
|
+
readonly open: string
|
|
156
|
+
readonly close: string
|
|
157
|
+
readonly raw: boolean
|
|
158
|
+
readonly strict: boolean
|
|
159
|
+
readonly ownSkip: readonly Combinator<unknown>[]
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export type ExecutableLexBody =
|
|
163
|
+
| ExecutableOptionalSuffixLexBody
|
|
164
|
+
| ExecutableTerminalLexBody
|
|
165
|
+
| ExecutableOrdered4LexBody
|
|
166
|
+
| ExecutableNot2LexBody
|
|
167
|
+
| ExecutableFixedLexBody
|
|
168
|
+
| ExecutableBalancedLexBody
|
|
169
|
+
|
|
170
|
+
export type LexBodyCandidate =
|
|
171
|
+
| { readonly strategy: 'character'; readonly estimatedOps: number }
|
|
172
|
+
| {
|
|
173
|
+
readonly strategy: 'token'
|
|
174
|
+
readonly estimatedOps: number
|
|
175
|
+
readonly body: OptionalSuffixLexBody
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Exact recognizer shape admitted by the first end-to-end Stage-C tranche. */
|
|
179
|
+
export function optionalSuffixLexBody(ir: LexicalIr): OptionalSuffixLexBody | undefined {
|
|
180
|
+
if (ir.kind !== 'sequence' || ir.parts.length !== 2) return undefined
|
|
181
|
+
const [base, suffix] = ir.parts
|
|
182
|
+
if (base?.kind !== 'regex' || suffix?.kind !== 'repeat'
|
|
183
|
+
|| suffix.min !== 0 || suffix.max !== 1 || !suffix.greedy
|
|
184
|
+
|| suffix.mode !== 'possessive' || suffix.body.kind !== 'literal'
|
|
185
|
+
|| suffix.body.caseInsensitive || suffix.body.value.length !== 1) return undefined
|
|
186
|
+
return {
|
|
187
|
+
kind: 'regex-optional-code-unit', source: base.source, flags: base.flags,
|
|
188
|
+
suffix: suffix.body.value,
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Compare two COMPLETE candidates. The CHARACTER cost is the existing
|
|
193
|
+
* token-boundary + sequence + regex + optional + literal body. The TOKEN cost
|
|
194
|
+
* is one selected body and one materialization boundary; neither candidate is
|
|
195
|
+
* serialized until this decision returns. */
|
|
196
|
+
export function selectOptionalSuffixLexBody(ir: LexicalIr): LexBodyCandidate {
|
|
197
|
+
const body = optionalSuffixLexBody(ir)
|
|
198
|
+
const character: LexBodyCandidate = { strategy: 'character', estimatedOps: 5 }
|
|
199
|
+
if (body === undefined) return character
|
|
200
|
+
const token: LexBodyCandidate = { strategy: 'token', estimatedOps: 2, body }
|
|
201
|
+
return token.estimatedOps < character.estimatedOps ? token : character
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Unwrap only scopes that token() has already installed identically. The
|
|
205
|
+
* returned parser is still the authored terminal/sequence; no effectful wrapper
|
|
206
|
+
* or final named winner is guessed from construction metadata. */
|
|
207
|
+
function directTokenBodyParser(
|
|
208
|
+
parser: Combinator<unknown>,
|
|
209
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
210
|
+
): Combinator<unknown> | undefined {
|
|
211
|
+
const outer = parser._def
|
|
212
|
+
if (outer.tag !== 'token') return undefined
|
|
213
|
+
let body = outer.parser
|
|
214
|
+
const seen = new Set<Combinator<unknown>>()
|
|
215
|
+
while (!seen.has(body)) {
|
|
216
|
+
seen.add(body)
|
|
217
|
+
const wrapper = body._def
|
|
218
|
+
if (wrapper.tag === 'grammar') {
|
|
219
|
+
if (wrapper.clearTrivia !== true || wrapper.triviaParser !== undefined
|
|
220
|
+
|| wrapper.captureTrivia === true || wrapper.rootCapture === 'opaque'
|
|
221
|
+
|| wrapper.constructionTrackLines !== 'inherit'
|
|
222
|
+
|| wrapper.constructionCaptureTriviaKinds !== undefined) return undefined
|
|
223
|
+
body = wrapper.parser
|
|
224
|
+
continue
|
|
225
|
+
}
|
|
226
|
+
if (wrapper.tag === 'lazy') {
|
|
227
|
+
const target = resolvedLazyTarget(body, resolve)
|
|
228
|
+
if (target === undefined) return undefined
|
|
229
|
+
body = target
|
|
230
|
+
continue
|
|
231
|
+
}
|
|
232
|
+
return body
|
|
233
|
+
}
|
|
234
|
+
return undefined
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Refuse wrapper/effect lookalikes even when normalization erases them to the
|
|
238
|
+
* same language. This first executable body owns only the authored token
|
|
239
|
+
* boundary around a direct sequence(regex, optional(literal-one-code-unit)). */
|
|
240
|
+
export function directOptionalSuffixTokenBody(
|
|
241
|
+
parser: Combinator<unknown>,
|
|
242
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
243
|
+
): ExecutableOptionalSuffixLexBody | undefined {
|
|
244
|
+
const inner = directTokenBodyParser(parser, resolve)
|
|
245
|
+
if (inner === undefined) return undefined
|
|
246
|
+
const sequence = inner._def
|
|
247
|
+
if (sequence.tag !== 'sequence' || sequence.parsers.length !== 2) return undefined
|
|
248
|
+
let baseParser = sequence.parsers[0]
|
|
249
|
+
const seenBase = new Set<Combinator<unknown>>()
|
|
250
|
+
while (baseParser !== undefined && baseParser._def.tag === 'lazy' && !seenBase.has(baseParser)) {
|
|
251
|
+
seenBase.add(baseParser)
|
|
252
|
+
baseParser = resolvedLazyTarget(baseParser, resolve)
|
|
253
|
+
}
|
|
254
|
+
const base = baseParser?._def
|
|
255
|
+
const optional = sequence.parsers[1]?._def
|
|
256
|
+
if (base?.tag !== 'regex' || optional?.tag !== 'optional') return undefined
|
|
257
|
+
const suffix = optional.parser._def
|
|
258
|
+
if (suffix.tag !== 'literal' || suffix.caseInsensitive
|
|
259
|
+
|| suffix.value.length !== 1) return undefined
|
|
260
|
+
const selected = selectOptionalSuffixLexBody({
|
|
261
|
+
kind: 'sequence',
|
|
262
|
+
parts: [
|
|
263
|
+
{ kind: 'regex', source: base.source, flags: base.flags },
|
|
264
|
+
{
|
|
265
|
+
kind: 'repeat',
|
|
266
|
+
body: { kind: 'literal', value: suffix.value, caseInsensitive: false },
|
|
267
|
+
min: 0, max: 1, greedy: true, mode: 'possessive',
|
|
268
|
+
},
|
|
269
|
+
],
|
|
270
|
+
})
|
|
271
|
+
return selected.strategy === 'token' ? {
|
|
272
|
+
...selected.body,
|
|
273
|
+
baseCanMatchNewline: baseParser!._meta.canMatchNewline,
|
|
274
|
+
suffixCanMatchNewline: optional.parser._meta.canMatchNewline,
|
|
275
|
+
expected: directTerminalFailureExpected(base),
|
|
276
|
+
} : undefined
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Exact direct token terminal bodies. `keywords()` is represented by the same
|
|
280
|
+
* canonical sticky RegExp its authored parser and OP_RX lowering use. Literal
|
|
281
|
+
* stays CHARACTER for now: case-insensitive literal() intentionally uses ASCII
|
|
282
|
+
* folding, which is not JavaScript RegExp `/i` semantics for non-ASCII text. */
|
|
283
|
+
export function directTerminalTokenBody(
|
|
284
|
+
parser: Combinator<unknown>,
|
|
285
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
286
|
+
): ExecutableTerminalLexBody | undefined {
|
|
287
|
+
const outer = parser._def
|
|
288
|
+
if (outer.tag !== 'token') return undefined
|
|
289
|
+
const body = directTokenBodyParser(parser, resolve)
|
|
290
|
+
if (body === undefined) return undefined
|
|
291
|
+
const def = body._def
|
|
292
|
+
if (def.tag === 'regex') return {
|
|
293
|
+
kind: 'regex-terminal', source: def.source, flags: def.flags,
|
|
294
|
+
canMatchNewline: body._meta.canMatchNewline,
|
|
295
|
+
expected: directTerminalFailureExpected(def),
|
|
296
|
+
}
|
|
297
|
+
if (def.tag !== 'keywords') return undefined
|
|
298
|
+
const re = keywordsRegExp(def.words, def.boundary, def.caseInsensitive)
|
|
299
|
+
return {
|
|
300
|
+
kind: 'regex-terminal', source: re.source, flags: re.flags,
|
|
301
|
+
canMatchNewline: body._meta.canMatchNewline,
|
|
302
|
+
expected: directTerminalFailureExpected(def),
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function directTerminalParser(
|
|
307
|
+
parser: Combinator<unknown>,
|
|
308
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
309
|
+
): Combinator<unknown> | undefined {
|
|
310
|
+
const seen = new Set<Combinator<unknown>>()
|
|
311
|
+
while (!seen.has(parser)) {
|
|
312
|
+
seen.add(parser)
|
|
313
|
+
const def = parser._def
|
|
314
|
+
if (def.tag === 'lazy') {
|
|
315
|
+
const target = resolvedLazyTarget(parser, resolve)
|
|
316
|
+
if (target === undefined) return undefined
|
|
317
|
+
parser = target
|
|
318
|
+
continue
|
|
319
|
+
}
|
|
320
|
+
if (def.tag === 'token') { parser = def.parser; continue }
|
|
321
|
+
// A grammar scope is not a recognition-only wrapper even when it clears
|
|
322
|
+
// trivia: it parses on a cloned context and owns line annotation plus
|
|
323
|
+
// failure-register publication. A childless composite body cannot erase
|
|
324
|
+
// that transaction. Keep the whole token on CHARACTER lowering instead.
|
|
325
|
+
if (def.tag === 'grammar') return undefined
|
|
326
|
+
return parser
|
|
327
|
+
}
|
|
328
|
+
return undefined
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function directTerminalLexBody(
|
|
332
|
+
parser: Combinator<unknown>,
|
|
333
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
334
|
+
): ExecutableTerminalLexBody | undefined {
|
|
335
|
+
const terminal = directTerminalParser(parser, resolve)
|
|
336
|
+
if (terminal === undefined) return undefined
|
|
337
|
+
const def = terminal._def
|
|
338
|
+
if (def.tag === 'literal') {
|
|
339
|
+
if (def.caseInsensitive) return undefined
|
|
340
|
+
const source = def.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
341
|
+
return {
|
|
342
|
+
kind: 'regex-terminal', source, flags: '',
|
|
343
|
+
canMatchNewline: terminal._meta.canMatchNewline,
|
|
344
|
+
expected: directTerminalFailureExpected(def),
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
if (def.tag === 'regex') return {
|
|
348
|
+
kind: 'regex-terminal', source: def.source, flags: def.flags,
|
|
349
|
+
canMatchNewline: terminal._meta.canMatchNewline,
|
|
350
|
+
expected: directTerminalFailureExpected(def),
|
|
351
|
+
}
|
|
352
|
+
if (def.tag !== 'keywords') return undefined
|
|
353
|
+
const re = keywordsRegExp(def.words, def.boundary, def.caseInsensitive)
|
|
354
|
+
return {
|
|
355
|
+
kind: 'regex-terminal', source: re.source, flags: re.flags,
|
|
356
|
+
canMatchNewline: terminal._meta.canMatchNewline,
|
|
357
|
+
expected: directTerminalFailureExpected(def),
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** First composite selected body: the hot Less at-keyword choice. Keep this a
|
|
362
|
+
* fixed template, not a recursive lexical interpreter. Equal finite first sets
|
|
363
|
+
* are load-bearing: at a matching lead all four arms really run in source
|
|
364
|
+
* order, while an excluded lead contributes only the choice's static opener
|
|
365
|
+
* set and no probe event. */
|
|
366
|
+
export function directOrdered4TokenBody(
|
|
367
|
+
parser: Combinator<unknown>,
|
|
368
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
369
|
+
): ExecutableOrdered4LexBody | undefined {
|
|
370
|
+
const body = directTokenBodyParser(parser, resolve)
|
|
371
|
+
return body === undefined ? undefined : directOrdered4Body(body, resolve)
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function directOrdered4Body(
|
|
375
|
+
body: Combinator<unknown>,
|
|
376
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
377
|
+
): ExecutableOrdered4LexBody | undefined {
|
|
378
|
+
if (body === undefined) return undefined
|
|
379
|
+
const choice = body._def
|
|
380
|
+
if (choice.tag !== 'choice' || choice.parsers.length !== 4
|
|
381
|
+
|| choice.strategy.tag !== 'firstMatch'
|
|
382
|
+
|| choice.gates.some(gate => gate !== null)
|
|
383
|
+
|| choice.autoNot.some(check => check !== null)) return undefined
|
|
384
|
+
|
|
385
|
+
const arms: ExecutableTerminalLexBody[] = []
|
|
386
|
+
let common: FirstSet | undefined
|
|
387
|
+
for (const arm of choice.parsers) {
|
|
388
|
+
const selected = directTerminalLexBody(arm, resolve)
|
|
389
|
+
if (selected === undefined) return undefined
|
|
390
|
+
if (selected.canMatchNewline) return undefined
|
|
391
|
+
const terminal = directTerminalParser(arm, resolve)!
|
|
392
|
+
const first = firstSetOf(terminal, new Set(), resolve)
|
|
393
|
+
if (first.kind !== 'ranges' || first.ranges.length === 0) return undefined
|
|
394
|
+
if (common === undefined) common = first
|
|
395
|
+
else if (JSON.stringify(common) !== JSON.stringify(first)) return undefined
|
|
396
|
+
arms.push(selected)
|
|
397
|
+
}
|
|
398
|
+
return {
|
|
399
|
+
kind: 'ordered4-terminals', commonFirstSet: common!,
|
|
400
|
+
commonMissExpected: deriveExpected(body),
|
|
401
|
+
expected: arms.flatMap(arm => arm.expected),
|
|
402
|
+
arms: arms as unknown as ExecutableOrdered4LexBody['arms'],
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export function directNot2TokenBody(
|
|
407
|
+
parser: Combinator<unknown>,
|
|
408
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
409
|
+
): ExecutableNot2LexBody | undefined {
|
|
410
|
+
const body = directTokenBodyParser(parser, resolve)
|
|
411
|
+
const sequence = body?._def
|
|
412
|
+
if (sequence?.tag !== 'sequence' || sequence.parsers.length !== 3) return undefined
|
|
413
|
+
const [guard0Parser, guard1Parser, terminalParser] = sequence.parsers
|
|
414
|
+
const guard0Def = guard0Parser?._def
|
|
415
|
+
const guard1Def = guard1Parser?._def
|
|
416
|
+
if (guard0Def?.tag !== 'not' || guard1Def?.tag !== 'not') return undefined
|
|
417
|
+
const ordered = directTokenBodyParser(guard0Def.parser, resolve)
|
|
418
|
+
?? directTerminalParser(guard0Def.parser, resolve)
|
|
419
|
+
const guard0Body = ordered === undefined ? undefined : directOrdered4Body(ordered, resolve)
|
|
420
|
+
const guard1 = directTerminalLexBody(guard1Def.parser, resolve)
|
|
421
|
+
const terminal = terminalParser === undefined ? undefined : directTerminalLexBody(terminalParser, resolve)
|
|
422
|
+
if (guard0Body === undefined || guard1 === undefined || terminal === undefined
|
|
423
|
+
|| guard1.canMatchNewline || terminal.canMatchNewline) return undefined
|
|
424
|
+
return {
|
|
425
|
+
kind: 'not2-ordered4-terminal',
|
|
426
|
+
guard0Expected: assertionFailureExpected(false, guard0Def.parser._tag),
|
|
427
|
+
guard1Expected: assertionFailureExpected(false, guard1Def.parser._tag),
|
|
428
|
+
terminalExpected: terminal.expected,
|
|
429
|
+
guard0: guard0Body.arms,
|
|
430
|
+
guard1, terminal,
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function directFixedLexNode(
|
|
435
|
+
parser: Combinator<unknown>,
|
|
436
|
+
resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
|
|
437
|
+
active: Set<Combinator<unknown>>,
|
|
438
|
+
pureOnly = false,
|
|
439
|
+
): ExecutableLexNode | undefined {
|
|
440
|
+
let current = parser
|
|
441
|
+
const wrappers = new Set<Combinator<unknown>>()
|
|
442
|
+
while (!wrappers.has(current)) {
|
|
443
|
+
wrappers.add(current)
|
|
444
|
+
const def = current._def
|
|
445
|
+
if (def.tag === 'lazy') {
|
|
446
|
+
const target = resolvedLazyTarget(current, resolve)
|
|
447
|
+
if (target === undefined) return undefined
|
|
448
|
+
current = target
|
|
449
|
+
continue
|
|
450
|
+
}
|
|
451
|
+
if (def.tag === 'token') { current = def.parser; continue }
|
|
452
|
+
break
|
|
453
|
+
}
|
|
454
|
+
if (wrappers.has(current) && current !== parser && current._def.tag === 'lazy') return undefined
|
|
455
|
+
if (active.has(current)) return undefined
|
|
456
|
+
active.add(current)
|
|
457
|
+
try {
|
|
458
|
+
const terminal = directTerminalLexBody(current, resolve)
|
|
459
|
+
if (terminal !== undefined) return { kind: 'terminal', terminal }
|
|
460
|
+
const def = current._def
|
|
461
|
+
if (def.tag === 'grammar') {
|
|
462
|
+
if (!pureOnly || (def.parser._def.tag !== 'token'
|
|
463
|
+
&& (def.clearTrivia !== true || def.triviaParser !== undefined
|
|
464
|
+
|| def.captureTrivia === true || def.rootCapture === 'opaque'))) return undefined
|
|
465
|
+
return directFixedLexNode(def.parser, resolve, active, true)
|
|
466
|
+
}
|
|
467
|
+
if (def.tag === 'sequence') {
|
|
468
|
+
if (def.parsers.length < 2 || def.parsers.length > 5) return undefined
|
|
469
|
+
const parts = def.parsers.map(part => directFixedLexNode(part, resolve, active, pureOnly))
|
|
470
|
+
if (parts.some(part => part === undefined)) return undefined
|
|
471
|
+
return { kind: 'sequence', parts: parts as ExecutableLexNode[] }
|
|
472
|
+
}
|
|
473
|
+
if (def.tag === 'choice') {
|
|
474
|
+
if ((def.parsers.length !== 2 && def.parsers.length !== 4 && def.parsers.length !== 8)
|
|
475
|
+
|| def.strategy.tag !== 'firstMatch'
|
|
476
|
+
|| def.gates.some(gate => gate !== null)
|
|
477
|
+
|| def.autoNot.some(check => check !== null)) return undefined
|
|
478
|
+
const arms = def.parsers.map(arm => directFixedLexNode(arm, resolve, active, pureOnly))
|
|
479
|
+
if (arms.some(arm => arm === undefined)) return undefined
|
|
480
|
+
const final = classifyFinalChoice(def.parsers, resolve)
|
|
481
|
+
return {
|
|
482
|
+
kind: 'choice', arms: arms as ExecutableLexNode[],
|
|
483
|
+
firstSets: final.firstSets,
|
|
484
|
+
armExpected: def.parsers.map(deriveExpected),
|
|
485
|
+
expected: def.parsers.flatMap(deriveExpected),
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
if (def.tag === 'not') {
|
|
489
|
+
const body = directFixedLexNode(def.parser, resolve, active, true)
|
|
490
|
+
return body === undefined ? undefined : {
|
|
491
|
+
kind: 'not', body,
|
|
492
|
+
expected: assertionFailureExpected(false, def.parser._tag),
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
if (def.tag === 'optional') {
|
|
496
|
+
const body = directFixedLexNode(def.parser, resolve, active, pureOnly)
|
|
497
|
+
return body === undefined ? undefined : { kind: 'optional', body }
|
|
498
|
+
}
|
|
499
|
+
return undefined
|
|
500
|
+
} finally {
|
|
501
|
+
active.delete(current)
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
export function directFixedTokenBody(
|
|
506
|
+
parser: Combinator<unknown>,
|
|
507
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
508
|
+
): ExecutableFixedLexBody | undefined {
|
|
509
|
+
const body = directTokenBodyParser(parser, resolve)
|
|
510
|
+
if (body === undefined) return undefined
|
|
511
|
+
const root = directFixedLexNode(body, resolve, new Set())
|
|
512
|
+
return root === undefined ? undefined : { kind: 'fixed-tree', root }
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
export function directBalancedTokenBody(
|
|
516
|
+
parser: Combinator<unknown>,
|
|
517
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
518
|
+
): ExecutableBalancedLexBody | undefined {
|
|
519
|
+
if (directTokenBodyParser(parser, resolve) === undefined) return undefined
|
|
520
|
+
const spec = balancedSpecOf(parser)
|
|
521
|
+
return spec === undefined ? undefined : {
|
|
522
|
+
kind: 'balanced-scan', open: spec.open, close: spec.close,
|
|
523
|
+
raw: spec.raw, strict: spec.strict, ownSkip: spec.ownSkip,
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
export function directExecutableTokenBody(
|
|
528
|
+
parser: Combinator<unknown>,
|
|
529
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
530
|
+
): ExecutableLexBody | undefined {
|
|
531
|
+
return directOptionalSuffixTokenBody(parser, resolve)
|
|
532
|
+
?? directTerminalTokenBody(parser, resolve)
|
|
533
|
+
?? directOrdered4TokenBody(parser, resolve)
|
|
534
|
+
?? directNot2TokenBody(parser, resolve)
|
|
535
|
+
?? directBalancedTokenBody(parser, resolve)
|
|
536
|
+
?? directFixedTokenBody(parser, resolve)
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** One canonical recognizer spec, shared by every family with equal lexical IR. */
|
|
540
|
+
export type LexicalRecognizer = {
|
|
541
|
+
readonly id: number
|
|
542
|
+
readonly key: string
|
|
543
|
+
readonly ir: LexicalIr
|
|
544
|
+
/** Crosswalk into the compiler-wide canonical recognition-family interner. */
|
|
545
|
+
readonly capabilityFamilyId: number
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Spelling-specific failure/probe contract. Families may share recognition IR
|
|
550
|
+
* without sharing where a structured spelling fails or what it expects.
|
|
551
|
+
*/
|
|
552
|
+
export type LexicalDiagnosticPlan = {
|
|
553
|
+
readonly id: number
|
|
554
|
+
readonly body: Combinator<unknown>
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/** Pointer-free, compiler-only checkpoints inside one authored token body. */
|
|
558
|
+
export type LexicalDiagnosticEvent =
|
|
559
|
+
| { readonly op: 'FAIL'; readonly state: number; readonly expectedId: number }
|
|
560
|
+
| {
|
|
561
|
+
readonly op: 'ASSERT'
|
|
562
|
+
readonly state: number
|
|
563
|
+
readonly innerStart: number
|
|
564
|
+
readonly innerEnd: number
|
|
565
|
+
readonly positive: boolean
|
|
566
|
+
readonly expectedId: number
|
|
567
|
+
readonly snapshotPolicy: 'saveLookaheadMark'
|
|
568
|
+
readonly execution: 'deferred'
|
|
569
|
+
}
|
|
570
|
+
| {
|
|
571
|
+
readonly op: 'REQUIRE'
|
|
572
|
+
readonly state: number
|
|
573
|
+
readonly childStart: number
|
|
574
|
+
readonly childEnd: number
|
|
575
|
+
readonly expectedId: number
|
|
576
|
+
readonly when: 'mandatory-iteration-2-to-min'
|
|
577
|
+
readonly committedChild: 'propagate'
|
|
578
|
+
readonly probe: 'child-only'
|
|
579
|
+
readonly anchor: 'repeat-position'
|
|
580
|
+
}
|
|
581
|
+
| {
|
|
582
|
+
readonly op: 'BAL_CLOSE_STRICT'
|
|
583
|
+
readonly state: number
|
|
584
|
+
readonly expectedId: number
|
|
585
|
+
readonly probe: true
|
|
586
|
+
readonly committed: false
|
|
587
|
+
readonly when: 'close-miss-after-open-body-success'
|
|
588
|
+
}
|
|
589
|
+
| {
|
|
590
|
+
readonly op: 'BAL_CLOSE_RECOVER'
|
|
591
|
+
readonly state: number
|
|
592
|
+
readonly expectedId: number
|
|
593
|
+
readonly probe: true
|
|
594
|
+
readonly committed: false
|
|
595
|
+
readonly when: 'close-miss-after-open-body-success'
|
|
596
|
+
readonly result: 'parseError'
|
|
597
|
+
readonly errorSpan: 'close-position'
|
|
598
|
+
readonly lineAnnotation: 'when-active'
|
|
599
|
+
readonly errorSink: 'when-active'
|
|
600
|
+
readonly cstErrorCapture: 'when-active'
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Stage-B2 diagnostic metadata. It is intentionally absent from TableProgram:
|
|
605
|
+
* assertion execution and every token-boundary effect remain capability GAPs.
|
|
606
|
+
*/
|
|
607
|
+
export type LexicalTransitionDiagnosticPlan = {
|
|
608
|
+
readonly id: number
|
|
609
|
+
readonly stateCount: number
|
|
610
|
+
readonly expected: readonly (readonly string[])[]
|
|
611
|
+
readonly events: readonly LexicalDiagnosticEvent[]
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
export type LexicalControlNode = {
|
|
615
|
+
readonly id: number
|
|
616
|
+
readonly parentControlId?: number
|
|
617
|
+
/** Authored parser kind, or the conditional/repeated balanced skipper role. */
|
|
618
|
+
readonly kind: ParserDef['tag'] | 'balanced' | 'balanced-skip'
|
|
619
|
+
readonly stateStart: number
|
|
620
|
+
readonly stateEnd: number
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
export type LexicalControlPlan = {
|
|
624
|
+
readonly id: number
|
|
625
|
+
/** Pointer-free source-control index over the one normalization session. */
|
|
626
|
+
readonly controls: readonly LexicalControlNode[]
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** One globally interned lexical family/base-token id. */
|
|
630
|
+
export type LexicalTokenFamily = {
|
|
631
|
+
readonly id: number
|
|
632
|
+
readonly recognizerId: number
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
/** One authored `token()` site, with spelling-specific diagnostics/effects. */
|
|
636
|
+
export type LexicalTokenSite = {
|
|
637
|
+
readonly parser: Combinator<unknown>
|
|
638
|
+
readonly body: Combinator<unknown>
|
|
639
|
+
/** Absent on a refused site; refused metadata allocates no diagnostic plan. */
|
|
640
|
+
readonly diagnosticId?: number
|
|
641
|
+
/** All three are absent when normalization declined. */
|
|
642
|
+
readonly familyId?: number
|
|
643
|
+
readonly recognizerId?: number
|
|
644
|
+
readonly refusal?: string
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
export type LexicalCapabilityStatus =
|
|
648
|
+
| { readonly kind: 'complete' }
|
|
649
|
+
| { readonly kind: 'gap'; readonly reason: string }
|
|
650
|
+
| { readonly kind: 'impossible'; readonly proof: string }
|
|
651
|
+
|
|
652
|
+
export type LexicalCapabilityPhase = {
|
|
653
|
+
readonly representation: LexicalCapabilityStatus
|
|
654
|
+
/** A complete body is constructible for every required reader/variant before selection. */
|
|
655
|
+
readonly executableLowering: LexicalCapabilityStatus
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
export type LexicalCapabilityObligations = {
|
|
659
|
+
readonly recognition: LexicalCapabilityPhase
|
|
660
|
+
readonly diagnostics: LexicalCapabilityPhase
|
|
661
|
+
readonly boundaryPlan: LexicalCapabilityPhase
|
|
662
|
+
readonly materializationPlan: LexicalCapabilityPhase
|
|
663
|
+
readonly supportedVariants: LexicalCapabilityPhase
|
|
664
|
+
readonly bindingAndReachability: LexicalCapabilityPhase
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/** Built-in authored token() context transaction. Compiler-only in this tranche. */
|
|
668
|
+
export type LexicalBoundaryPlan = {
|
|
669
|
+
readonly id: 0
|
|
670
|
+
readonly kind: 'token-context-transaction'
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/** Built-in authored token() source-range value/CST materialization. */
|
|
674
|
+
export type LexicalMaterializationPlan = {
|
|
675
|
+
readonly id: 0
|
|
676
|
+
readonly kind: 'token-source-range'
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** Exact construction-time grammar wrapper policy, not merely its effective context. */
|
|
680
|
+
export type LexicalGrammarWrapperSpec = {
|
|
681
|
+
readonly id: number
|
|
682
|
+
readonly sourceOperationId: number
|
|
683
|
+
readonly clearTrivia: boolean
|
|
684
|
+
readonly triviaBindingId?: number
|
|
685
|
+
readonly trackLines: 'on' | 'off' | 'inherit'
|
|
686
|
+
readonly captureTrivia: boolean
|
|
687
|
+
readonly captureTriviaKindsId?: number
|
|
688
|
+
readonly rootCapture: 'opaque' | 'inherit'
|
|
689
|
+
readonly clonePolicy: 'spread-existing-or-create-canonical'
|
|
690
|
+
readonly postChildPolicy: 'line-propagate-then-annotate-or-return'
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/** Well-nested overlay over the sole LexicalIr/B2 control/state authority. */
|
|
694
|
+
export type LexicalWrapperFrame =
|
|
695
|
+
| {
|
|
696
|
+
readonly id: number
|
|
697
|
+
readonly parentFrameId?: number
|
|
698
|
+
readonly controlId: number
|
|
699
|
+
readonly stateStart: number
|
|
700
|
+
readonly stateEnd: number
|
|
701
|
+
readonly kind: 'token'
|
|
702
|
+
readonly boundaryPlanId: 0
|
|
703
|
+
readonly materializationPlanId: 0
|
|
704
|
+
}
|
|
705
|
+
| {
|
|
706
|
+
readonly id: number
|
|
707
|
+
readonly parentFrameId?: number
|
|
708
|
+
readonly controlId: number
|
|
709
|
+
readonly stateStart: number
|
|
710
|
+
readonly stateEnd: number
|
|
711
|
+
readonly kind: 'grammar'
|
|
712
|
+
readonly wrapperSpecId: number
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
export type LexicalBoundaryTopology = {
|
|
716
|
+
readonly id: number
|
|
717
|
+
/** Source-order outer-before-inner frames; recognition/control is not duplicated. */
|
|
718
|
+
readonly frames: readonly LexicalWrapperFrame[]
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
export type LexicalCapabilityContext = {
|
|
722
|
+
readonly trivia: Combinator<unknown> | undefined
|
|
723
|
+
readonly scanSkip: readonly Combinator<unknown>[]
|
|
724
|
+
readonly trackLines: boolean
|
|
725
|
+
readonly captureTrivia: boolean
|
|
726
|
+
readonly opaqueRootCapture: boolean
|
|
727
|
+
readonly dynamicState: boolean
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/** Immutable compiler occurrence context. Live scanner callbacks are policy,
|
|
731
|
+
* never identity-bearing lexical facts or pending-range keys. */
|
|
732
|
+
export type LexicalContextSnapshot = {
|
|
733
|
+
readonly id: number
|
|
734
|
+
readonly key: string
|
|
735
|
+
readonly hasTrivia: boolean
|
|
736
|
+
readonly hasScanSkip: boolean
|
|
737
|
+
readonly trackLines: boolean
|
|
738
|
+
readonly captureTrivia: boolean
|
|
739
|
+
readonly opaqueRootCapture: boolean
|
|
740
|
+
readonly dynamicState: boolean
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
export type LexicalOwnSkipEntry = {
|
|
744
|
+
readonly semanticKey?: string
|
|
745
|
+
readonly ir?: LexicalIr
|
|
746
|
+
readonly status: LexicalCapabilityStatus
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/** Construction-owned skipper children only. Ambient trivia/scanSkip never
|
|
750
|
+
* enter this pool because their object and `.parse` lookup are observable. */
|
|
751
|
+
export type LexicalOwnSkipPlan = {
|
|
752
|
+
readonly id: number
|
|
753
|
+
readonly entries: readonly LexicalOwnSkipEntry[]
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
export type LexicalScanConsumerPolicy =
|
|
757
|
+
| {
|
|
758
|
+
readonly id: number
|
|
759
|
+
readonly siteId: number
|
|
760
|
+
readonly path: string
|
|
761
|
+
readonly contextSnapshotId: number
|
|
762
|
+
readonly kind: 'scanTo'
|
|
763
|
+
readonly raw: boolean
|
|
764
|
+
readonly ambient: 'enumerate-trivia-then-scanSkip-per-parse-attempt' | 'none-raw'
|
|
765
|
+
readonly lookup: 'dynamic-parse-property-every-skip-test'
|
|
766
|
+
readonly context: 'detached-per-attempt-state-errors-only'
|
|
767
|
+
readonly ownSkipPlanId?: number
|
|
768
|
+
readonly pendingReuse: 'forbidden'
|
|
769
|
+
readonly status: LexicalCapabilityStatus
|
|
770
|
+
}
|
|
771
|
+
| {
|
|
772
|
+
readonly id: number
|
|
773
|
+
readonly siteId: number
|
|
774
|
+
readonly path: string
|
|
775
|
+
readonly contextSnapshotId: number
|
|
776
|
+
readonly kind: 'balanced'
|
|
777
|
+
readonly raw: boolean
|
|
778
|
+
readonly ambient: 'array-identity-interior-cache' | 'none-raw'
|
|
779
|
+
readonly cache: 'lookup-every-attempt-enumerate-on-miss'
|
|
780
|
+
readonly lookup: 'dynamic-parse-property-every-skip-test'
|
|
781
|
+
readonly context: 'token-cleared-original'
|
|
782
|
+
readonly ownSkipPlanId?: number
|
|
783
|
+
readonly pendingReuse: 'forbidden'
|
|
784
|
+
readonly status: LexicalCapabilityStatus
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/**
|
|
788
|
+
* Compiler-only phase-A census record. These ids are deliberately not family
|
|
789
|
+
* ids: two authored sites can share one future language while remaining two
|
|
790
|
+
* independently auditable capability obligations.
|
|
791
|
+
*/
|
|
792
|
+
export type LexicalCapabilitySite = {
|
|
793
|
+
readonly id: number
|
|
794
|
+
/** Final-graph compiler body identity. Never serialized as a parser pointer. */
|
|
795
|
+
readonly bodyId: number
|
|
796
|
+
/** Interned recognition-language summary; never used as a completeness id. */
|
|
797
|
+
readonly languageId: number
|
|
798
|
+
readonly path: string
|
|
799
|
+
/** Effective lexical scope at this occurrence; compiler-only, never a family id. */
|
|
800
|
+
readonly contextKey: string
|
|
801
|
+
readonly contextSnapshotId: number
|
|
802
|
+
/** Context seen by recognition after token() clears trivia/capture sinks. */
|
|
803
|
+
readonly recognitionContextKey: string
|
|
804
|
+
readonly context: LexicalCapabilityContext
|
|
805
|
+
readonly recognitionContext: LexicalCapabilityContext
|
|
806
|
+
readonly semanticKey: string
|
|
807
|
+
readonly atom: 'terminal' | 'token' | 'choice' | 'dispatch'
|
|
808
|
+
readonly parser: Combinator<unknown>
|
|
809
|
+
readonly obligations: LexicalCapabilityObligations
|
|
810
|
+
/** Internal body metadata only; never claims the boundary effects obligation. */
|
|
811
|
+
readonly diagnosticPlanId?: number
|
|
812
|
+
/** Compiler-only boundary overlay; absent when exact representation declined. */
|
|
813
|
+
readonly boundaryTopologyId?: number
|
|
814
|
+
/** Control ancestry authority used by the boundary overlay. */
|
|
815
|
+
readonly controlPlanId?: number
|
|
816
|
+
/** Scan consumers owned by this atomic occurrence, in structural order. */
|
|
817
|
+
readonly scanConsumerPolicyIds: readonly number[]
|
|
818
|
+
/** Derived from `obligations`; callers cannot independently set it. */
|
|
819
|
+
readonly status: LexicalCapabilityStatus
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
export type LexicalCapabilityLanguage = {
|
|
823
|
+
readonly id: number
|
|
824
|
+
readonly atom: LexicalCapabilitySite['atom']
|
|
825
|
+
readonly semanticKey: string
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
/**
|
|
829
|
+
* One compiler-wide canonical source-range language. IDs use the lexical
|
|
830
|
+
* family namespace even while phase B is disabled; decision plans and the
|
|
831
|
+
* legacy token collector cross-reference this one IR interner.
|
|
832
|
+
*/
|
|
833
|
+
export type LexicalDecisionFamily = {
|
|
834
|
+
readonly id: number
|
|
835
|
+
readonly semanticKey: string
|
|
836
|
+
readonly ir: LexicalIr
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/**
|
|
840
|
+
* A compatible view of one family range. `prefix` owns its own shorter end;
|
|
841
|
+
* consumers must never substitute the full family end (the `a | ab` rule).
|
|
842
|
+
*/
|
|
843
|
+
export type LexicalDecisionOutcomeView =
|
|
844
|
+
| { readonly kind: 'whole'; readonly relation: 'equal' }
|
|
845
|
+
| {
|
|
846
|
+
readonly kind: 'predicate'
|
|
847
|
+
readonly relation: 'equal'
|
|
848
|
+
readonly match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>
|
|
849
|
+
}
|
|
850
|
+
| {
|
|
851
|
+
readonly kind: 'language'
|
|
852
|
+
readonly relation: 'equal' | 'prefix'
|
|
853
|
+
readonly ir: LexicalIr
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/** Global, family-qualified atomic outcome/view identity. */
|
|
857
|
+
export type LexicalDecisionOutcome = {
|
|
858
|
+
readonly id: number
|
|
859
|
+
readonly familyId: number
|
|
860
|
+
readonly semanticKey: string
|
|
861
|
+
readonly view: LexicalDecisionOutcomeView
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
export type LexicalDecisionAcceptance =
|
|
865
|
+
| { readonly kind: 'outcomes'; readonly outcomeIds: readonly number[] }
|
|
866
|
+
| { readonly kind: 'otherwise'; readonly excludingOutcomeIds: readonly number[] }
|
|
867
|
+
| { readonly kind: 'unrestricted' }
|
|
868
|
+
| { readonly kind: 'impossible' }
|
|
869
|
+
| { readonly kind: 'gap'; readonly reason: string }
|
|
870
|
+
|
|
871
|
+
/** One source-ordered arm/route for one candidate family at one occurrence. */
|
|
872
|
+
export type LexicalDecisionArm = {
|
|
873
|
+
readonly armId: number
|
|
874
|
+
readonly acceptance: LexicalDecisionAcceptance
|
|
875
|
+
readonly usesRouted: boolean
|
|
876
|
+
readonly dynamicGate: boolean
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
export type LexicalDecisionFamilyPlan = {
|
|
880
|
+
readonly familyId: number
|
|
881
|
+
readonly arms: readonly LexicalDecisionArm[]
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** Source-ordered decision regions within which a recognized range may remain
|
|
885
|
+
* live. A dynamic scanner is its own closed epoch: callbacks and ambient skip
|
|
886
|
+
* policy may mutate parse state, so no lexical fact may cross that arm in
|
|
887
|
+
* either direction. This is compiler evidence for one complete TOKEN body,
|
|
888
|
+
* never a request to replay the CHARACTER body. */
|
|
889
|
+
export type LexicalDecisionReuseEpoch = {
|
|
890
|
+
readonly armIds: readonly number[]
|
|
891
|
+
readonly pendingReuse: 'forbidden' | 'unproved'
|
|
892
|
+
readonly boundary: 'none' | 'dynamic-scan'
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/** Occurrence-local ordered decision proof; never serialized in this tranche. */
|
|
896
|
+
export type LexicalDecisionSite = {
|
|
897
|
+
readonly siteId: number
|
|
898
|
+
readonly atom: 'choice' | 'dispatch'
|
|
899
|
+
readonly path: string
|
|
900
|
+
readonly contextKey: string
|
|
901
|
+
readonly contextSnapshotId: number
|
|
902
|
+
readonly families: readonly LexicalDecisionFamilyPlan[]
|
|
903
|
+
/** Dispatch-only source-order routed policy. This remains Stage-A authority;
|
|
904
|
+
* the effect projection references the site rather than copying route bits. */
|
|
905
|
+
readonly routeUsesRouted?: readonly boolean[]
|
|
906
|
+
/** Families not proven at this occurrence remain admitted through this
|
|
907
|
+
* conservative relation. It is an explicit TOKEN-body route, never a request
|
|
908
|
+
* to replay the character parser. */
|
|
909
|
+
readonly fallback: 'unrestricted'
|
|
910
|
+
/** Missing inclusion/partition proofs reduce pruning precision only. Every
|
|
911
|
+
* authored arm still runs its own exact replacement recognizer in PEG order. */
|
|
912
|
+
readonly precisionNotes: readonly string[]
|
|
913
|
+
/** Dynamic scanner callbacks are effectful and can never publish a reusable
|
|
914
|
+
* cross-arm lexical fact. All other sites remain unproved in this tranche. */
|
|
915
|
+
readonly pendingReuse: 'forbidden' | 'unproved'
|
|
916
|
+
readonly reuseEpochs: readonly LexicalDecisionReuseEpoch[]
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
/** Final winner-resolved fixed choice classes, shared with table encoding. */
|
|
920
|
+
export type LexicalChoiceClassAuthority = {
|
|
921
|
+
readonly id: number
|
|
922
|
+
readonly armIds: readonly number[]
|
|
923
|
+
readonly firstSets: readonly FirstSet[]
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
/** A non-authored execution order such as longest literal first. */
|
|
927
|
+
export type LexicalChoiceExecutionOrder = {
|
|
928
|
+
readonly id: number
|
|
929
|
+
readonly armIds: readonly number[]
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/** Exact classified spelling to authored arm identity. */
|
|
933
|
+
export type LexicalClassifySpellingMap = {
|
|
934
|
+
readonly id: number
|
|
935
|
+
readonly entries: readonly { readonly spelling: string; readonly armId: number }[]
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/** Transform callback identities applied to an already recognized spelling. */
|
|
939
|
+
export type LexicalClassifyProjectionPlan = {
|
|
940
|
+
readonly id: number
|
|
941
|
+
readonly entries: readonly {
|
|
942
|
+
readonly armId: number
|
|
943
|
+
readonly transformIds: readonly number[]
|
|
944
|
+
readonly transformSourceIds: readonly (number | -1)[]
|
|
945
|
+
readonly transformCount: number
|
|
946
|
+
}[]
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
export type LexicalExpectedAuthority = {
|
|
950
|
+
readonly id: number
|
|
951
|
+
readonly values: readonly string[]
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
/** Compiler-local callback identity; never a TableProgram or public wire value. */
|
|
955
|
+
export type LexicalDecisionCallbackAuthority = {
|
|
956
|
+
readonly id: number
|
|
957
|
+
readonly callback:
|
|
958
|
+
| ((state: unknown) => boolean)
|
|
959
|
+
| ((value: unknown, span: { start: number; end: number }) => unknown)
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/** Named emitted projection for a callback; compiler-only until Stage C selects. */
|
|
963
|
+
export type LexicalDecisionSourceAuthority = {
|
|
964
|
+
readonly id: number
|
|
965
|
+
readonly source: string
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
/** Exact post-success rejection checks, interned without re-deriving them. */
|
|
969
|
+
export type LexicalAutoNotAuthority = {
|
|
970
|
+
readonly id: number
|
|
971
|
+
readonly checks: readonly AutoNotCheck[]
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
export type LexicalFinalDecisionAuthority =
|
|
975
|
+
| {
|
|
976
|
+
readonly id: number
|
|
977
|
+
readonly atom: 'choice'
|
|
978
|
+
readonly decisionSiteId: number
|
|
979
|
+
readonly authoredArmIds: readonly number[]
|
|
980
|
+
readonly mode:
|
|
981
|
+
| { readonly kind: 'ordered' }
|
|
982
|
+
| { readonly kind: 'exclusive'; readonly classAuthorityId: number }
|
|
983
|
+
| { readonly kind: 'longest'; readonly orderAuthorityId: number }
|
|
984
|
+
| {
|
|
985
|
+
readonly kind: 'classify'
|
|
986
|
+
readonly superArmId: number
|
|
987
|
+
readonly spellingMapAuthorityId: number
|
|
988
|
+
readonly projectionAuthorityId: number
|
|
989
|
+
}
|
|
990
|
+
/** Static total-miss set only for strategies that never consult dynamic gates. */
|
|
991
|
+
readonly staticMissExpectedAuthorityId?: number
|
|
992
|
+
}
|
|
993
|
+
| {
|
|
994
|
+
readonly id: number
|
|
995
|
+
readonly atom: 'dispatch'
|
|
996
|
+
readonly decisionSiteId: number
|
|
997
|
+
readonly selectorBindingEdgeId: number
|
|
998
|
+
readonly classifierAuthorityId: number
|
|
999
|
+
readonly orderedRouteIds: readonly number[]
|
|
1000
|
+
readonly noRouteExpectedAuthorityId: number
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
export type LexicalDecisionChildEffect = {
|
|
1004
|
+
readonly id: number
|
|
1005
|
+
readonly decisionSiteId: number
|
|
1006
|
+
readonly armId: number
|
|
1007
|
+
readonly childBindingEdgeId: number
|
|
1008
|
+
readonly expectedAuthorityId: number
|
|
1009
|
+
readonly gateCallbackId?: number
|
|
1010
|
+
readonly gateSourceId?: number
|
|
1011
|
+
readonly autoNotAuthorityId?: number
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
export type LexicalDecisionEffectProgram = {
|
|
1015
|
+
readonly id: number
|
|
1016
|
+
readonly decisionSiteId: number
|
|
1017
|
+
readonly finalDecisionAuthorityId: number
|
|
1018
|
+
readonly phase: LexicalCapabilityPhase
|
|
1019
|
+
readonly childEffectIds: readonly number[]
|
|
1020
|
+
/** Exact fixed-edge projections consumed by the three existing readers. */
|
|
1021
|
+
readonly childBindingProjectionIds: readonly number[]
|
|
1022
|
+
readonly referenceTemplateId: number
|
|
1023
|
+
readonly capturedTemplateId: number
|
|
1024
|
+
readonly namedTemplateId: number
|
|
1025
|
+
readonly readerMask: number
|
|
1026
|
+
readonly variantMask: number
|
|
1027
|
+
readonly semanticDigest: number
|
|
1028
|
+
readonly pendingRollbackPolicyId: 0
|
|
1029
|
+
readonly commitPolicyId: 0
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/** Compiler-only C2 pools. No member is a TableProgram or runtime wire field. */
|
|
1033
|
+
export type LexicalDecisionEffectInventory = {
|
|
1034
|
+
readonly finalDecisionAuthorities: readonly LexicalFinalDecisionAuthority[]
|
|
1035
|
+
readonly decisionEffects: readonly LexicalDecisionEffectProgram[]
|
|
1036
|
+
readonly decisionChildEffects: readonly LexicalDecisionChildEffect[]
|
|
1037
|
+
readonly choiceClassAuthorities: readonly LexicalChoiceClassAuthority[]
|
|
1038
|
+
readonly choiceExecutionOrders: readonly LexicalChoiceExecutionOrder[]
|
|
1039
|
+
readonly classifySpellingMaps: readonly LexicalClassifySpellingMap[]
|
|
1040
|
+
readonly classifyProjectionPlans: readonly LexicalClassifyProjectionPlan[]
|
|
1041
|
+
readonly decisionExpectedAuthorities: readonly LexicalExpectedAuthority[]
|
|
1042
|
+
readonly decisionCallbackAuthorities: readonly LexicalDecisionCallbackAuthority[]
|
|
1043
|
+
readonly decisionSourceAuthorities: readonly LexicalDecisionSourceAuthority[]
|
|
1044
|
+
readonly decisionAutoNotAuthorities: readonly LexicalAutoNotAuthority[]
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
/** One fixed incoming/root edge whose direct linked-body candidates remain owed. */
|
|
1048
|
+
export type LexicalBindingEdge = {
|
|
1049
|
+
readonly id: number
|
|
1050
|
+
readonly path: string
|
|
1051
|
+
readonly contextKey: string
|
|
1052
|
+
/** Effective context at the child occurrence, after the parent boundary. */
|
|
1053
|
+
readonly childContextKey: string
|
|
1054
|
+
readonly contextSnapshotId: number
|
|
1055
|
+
readonly parentTag: ParserDef['tag'] | 'root'
|
|
1056
|
+
readonly childTag: ParserDef['tag']
|
|
1057
|
+
readonly projectionId?: number
|
|
1058
|
+
readonly status: LexicalCapabilityStatus
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
/** Compiler-owned fixed-edge lowering. These are numeric construction facts,
|
|
1062
|
+
* not a parse-time micro-IR: each reader consumes only its own preselected
|
|
1063
|
+
* template and the common digest proves all three describe the same edge. */
|
|
1064
|
+
export type LexicalBindingProjection = {
|
|
1065
|
+
readonly id: number
|
|
1066
|
+
readonly edgeId: number
|
|
1067
|
+
readonly parentBodyId: number
|
|
1068
|
+
readonly childBodyId: number
|
|
1069
|
+
readonly childOrdinal: number
|
|
1070
|
+
readonly referenceOperand: number
|
|
1071
|
+
readonly capturedSlotId: number
|
|
1072
|
+
readonly capturedTemplateId: number
|
|
1073
|
+
readonly namedSymbolId: number
|
|
1074
|
+
readonly namedTemplateId: number
|
|
1075
|
+
/** bit 0 reference, bit 1 captured/CSP, bit 2 named/emitted. */
|
|
1076
|
+
readonly readerMask: number
|
|
1077
|
+
/** Strict, recovery, tracking, CST, probe, and coverage construction modes. */
|
|
1078
|
+
readonly variantMask: number
|
|
1079
|
+
readonly semanticDigest: number
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
/** One executable primitive-terminal candidate plus every fixed incoming edge.
|
|
1083
|
+
* This remains compiler-only until Stage C selects a complete semantic body. */
|
|
1084
|
+
export type LexicalTerminalProjection = {
|
|
1085
|
+
readonly id: number
|
|
1086
|
+
readonly siteId: number
|
|
1087
|
+
readonly bodyId: number
|
|
1088
|
+
readonly languageId: number
|
|
1089
|
+
readonly referenceBodyId: number
|
|
1090
|
+
readonly capturedTemplateId: number
|
|
1091
|
+
readonly namedSymbolId: number
|
|
1092
|
+
readonly incomingBindingProjectionIds: readonly number[]
|
|
1093
|
+
readonly readerMask: number
|
|
1094
|
+
readonly variantMask: number
|
|
1095
|
+
readonly semanticDigest: number
|
|
1096
|
+
readonly status: LexicalCapabilityStatus
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
export type LexicalOutcomeMatch =
|
|
1100
|
+
| { readonly kind: 'exact'; readonly values: readonly string[]; readonly caseInsensitive: boolean }
|
|
1101
|
+
| { readonly kind: 'startsWith' | 'endsWith'; readonly value: string; readonly caseInsensitive: boolean }
|
|
1102
|
+
| { readonly kind: 'matches'; readonly value: string; readonly flags: string; readonly caseInsensitive: boolean }
|
|
1103
|
+
| { readonly kind: 'otherwise'; readonly excluding: readonly Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[] }
|
|
1104
|
+
|
|
1105
|
+
/** A globally reusable classification id for one family/range predicate. */
|
|
1106
|
+
export type LexicalOutcomeSpec = {
|
|
1107
|
+
readonly id: number
|
|
1108
|
+
readonly familyId: number
|
|
1109
|
+
readonly match: LexicalOutcomeMatch
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
/** One compatible view over a family range at a particular dispatch site. */
|
|
1113
|
+
export type LexicalTokenOutcome = {
|
|
1114
|
+
readonly id: number
|
|
1115
|
+
readonly match: LexicalOutcomeMatch
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
/** One ordered dispatch arm. IDs classify; this route owns the branch/cut. */
|
|
1119
|
+
export type LexicalTokenRoute = {
|
|
1120
|
+
readonly index: number
|
|
1121
|
+
readonly kind: 'exact' | 'matcher' | 'otherwise'
|
|
1122
|
+
readonly acceptedIds: readonly number[]
|
|
1123
|
+
readonly matches: readonly LexicalOutcomeMatch[]
|
|
1124
|
+
readonly parser: Combinator<unknown>
|
|
1125
|
+
readonly usesRouted: boolean
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/** Dispatch outcomes stay site-local; ordered dispatch semantics are unchanged. */
|
|
1129
|
+
export type LexicalTokenClassifier = {
|
|
1130
|
+
readonly dispatch: Combinator<unknown>
|
|
1131
|
+
readonly familyId: number
|
|
1132
|
+
/** True when reaching token() crossed a wrapper whose effects must still run. */
|
|
1133
|
+
readonly selectorEffects: boolean
|
|
1134
|
+
/** Compatible range views, flattened and allowed to repeat an id by route. */
|
|
1135
|
+
readonly outcomes: readonly LexicalTokenOutcome[]
|
|
1136
|
+
/** Authoritative ordered route/cut identity. */
|
|
1137
|
+
readonly routes: readonly LexicalTokenRoute[]
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
export type LexicalAlphabet = {
|
|
1141
|
+
/**
|
|
1142
|
+
* Compiler-only graph metadata. `diagnostics`, `sites`, `classifiers`, and
|
|
1143
|
+
* `familyIdOf` contain live combinators/Maps and are NOT TableProgram data.
|
|
1144
|
+
* A serializer must project the numeric/IR specs and relocate site references.
|
|
1145
|
+
*/
|
|
1146
|
+
readonly recognizers: readonly LexicalRecognizer[]
|
|
1147
|
+
readonly diagnostics: readonly LexicalDiagnosticPlan[]
|
|
1148
|
+
readonly families: readonly LexicalTokenFamily[]
|
|
1149
|
+
readonly sites: readonly LexicalTokenSite[]
|
|
1150
|
+
readonly outcomes: readonly LexicalOutcomeSpec[]
|
|
1151
|
+
readonly classifiers: readonly LexicalTokenClassifier[]
|
|
1152
|
+
readonly familyIdOf: ReadonlyMap<Combinator<unknown>, number>
|
|
1153
|
+
/** Whole-final-grammar, ownership-aware phase-A capability census. */
|
|
1154
|
+
readonly capabilities: readonly LexicalCapabilitySite[]
|
|
1155
|
+
/** Interned language view only; completeness is always occurrence-based. */
|
|
1156
|
+
readonly capabilityLanguages: readonly LexicalCapabilityLanguage[]
|
|
1157
|
+
/** Every distinct fixed parent/root edge, independently GAP until linked. */
|
|
1158
|
+
readonly bindingEdges: readonly LexicalBindingEdge[]
|
|
1159
|
+
readonly bindingProjections: readonly LexicalBindingProjection[]
|
|
1160
|
+
readonly terminalProjections: readonly LexicalTerminalProjection[]
|
|
1161
|
+
/** Phase-A compatible range/arm algebra; compiler-only and occurrence-local. */
|
|
1162
|
+
readonly decisionFamilies: readonly LexicalDecisionFamily[]
|
|
1163
|
+
readonly decisionOutcomes: readonly LexicalDecisionOutcome[]
|
|
1164
|
+
readonly decisions: readonly LexicalDecisionSite[]
|
|
1165
|
+
readonly decisionEffectPlan: LexicalDecisionEffectInventory
|
|
1166
|
+
readonly transitionDiagnostics: readonly LexicalTransitionDiagnosticPlan[]
|
|
1167
|
+
readonly boundaryPlans: readonly LexicalBoundaryPlan[]
|
|
1168
|
+
readonly materializationPlans: readonly LexicalMaterializationPlan[]
|
|
1169
|
+
readonly grammarWrapperSpecs: readonly LexicalGrammarWrapperSpec[]
|
|
1170
|
+
readonly grammarCaptureTriviaKinds: readonly (readonly string[])[]
|
|
1171
|
+
readonly boundaryTopologies: readonly LexicalBoundaryTopology[]
|
|
1172
|
+
readonly controlPlans: readonly LexicalControlPlan[]
|
|
1173
|
+
readonly contextSnapshots: readonly LexicalContextSnapshot[]
|
|
1174
|
+
readonly ownSkipPlans: readonly LexicalOwnSkipPlan[]
|
|
1175
|
+
readonly scanConsumerPolicies: readonly LexicalScanConsumerPolicy[]
|
|
1176
|
+
/** False means phase B is forbidden for the entire program. */
|
|
1177
|
+
readonly capabilityComplete: boolean
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
export type LexicalCapabilityInventory = Pick<
|
|
1181
|
+
LexicalAlphabet,
|
|
1182
|
+
'capabilities' | 'capabilityLanguages' | 'bindingEdges'
|
|
1183
|
+
| 'bindingProjections' | 'terminalProjections'
|
|
1184
|
+
| 'decisionFamilies' | 'decisionOutcomes' | 'decisions' | 'decisionEffectPlan'
|
|
1185
|
+
| 'transitionDiagnostics'
|
|
1186
|
+
| 'boundaryPlans' | 'materializationPlans' | 'grammarWrapperSpecs'
|
|
1187
|
+
| 'grammarCaptureTriviaKinds' | 'boundaryTopologies'
|
|
1188
|
+
| 'controlPlans'
|
|
1189
|
+
| 'contextSnapshots' | 'ownSkipPlans' | 'scanConsumerPolicies'
|
|
1190
|
+
| 'capabilityComplete'
|
|
1191
|
+
>
|
|
1192
|
+
|
|
1193
|
+
/** Numeric-only artifact projection; `TableProgram` aliases this contract. */
|
|
1194
|
+
export type NumericLexicalPlan = {
|
|
1195
|
+
readonly recognizerOffsets: readonly number[]
|
|
1196
|
+
readonly recognizerData: readonly number[]
|
|
1197
|
+
readonly outcomeOffsets: readonly number[]
|
|
1198
|
+
readonly outcomeData: readonly number[]
|
|
1199
|
+
readonly tokenSites: readonly number[]
|
|
1200
|
+
readonly sites: readonly number[]
|
|
1201
|
+
readonly routes: readonly number[]
|
|
1202
|
+
readonly accepted: readonly number[]
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
/** Lexical-family ids occupy their own published namespace. */
|
|
1206
|
+
export const FIRST_LEXICAL_FAMILY_ID = 3
|
|
1207
|
+
|
|
1208
|
+
const foldAscii = (value: string): string => value.replace(/[A-Z]/g, c => c.toLowerCase())
|
|
1209
|
+
|
|
1210
|
+
/** Canonical, parser-free identity for one range predicate. */
|
|
1211
|
+
export function canonicalLexicalOutcomeKey(match: LexicalOutcomeMatch): string {
|
|
1212
|
+
if (match.kind === 'exact') {
|
|
1213
|
+
const values = [...new Set(match.values
|
|
1214
|
+
.map(value => match.caseInsensitive ? foldAscii(value) : value))].sort()
|
|
1215
|
+
return JSON.stringify({ kind: match.kind, values, caseInsensitive: match.caseInsensitive })
|
|
1216
|
+
}
|
|
1217
|
+
if (match.kind === 'startsWith' || match.kind === 'endsWith') {
|
|
1218
|
+
return JSON.stringify({
|
|
1219
|
+
kind: match.kind,
|
|
1220
|
+
value: match.caseInsensitive ? foldAscii(match.value) : match.value,
|
|
1221
|
+
caseInsensitive: match.caseInsensitive,
|
|
1222
|
+
})
|
|
1223
|
+
}
|
|
1224
|
+
if (match.kind === 'matches') {
|
|
1225
|
+
const rawFlags = match.caseInsensitive && !match.flags.includes('i') ? `${match.flags}i` : match.flags
|
|
1226
|
+
const flags = new RegExp('', rawFlags.replace(/g/g, '')).flags
|
|
1227
|
+
return JSON.stringify({ kind: match.kind, value: match.value, flags })
|
|
1228
|
+
}
|
|
1229
|
+
if (match.kind !== 'otherwise') throw new Error('parseman: unknown lexical outcome predicate')
|
|
1230
|
+
const excluding = match.excluding
|
|
1231
|
+
const fixed = excluding.filter((entry): entry is Extract<LexicalOutcomeMatch, { kind: 'startsWith' | 'endsWith' }> =>
|
|
1232
|
+
entry.kind === 'startsWith' || entry.kind === 'endsWith')
|
|
1233
|
+
const fixedContainsExact = (
|
|
1234
|
+
exact: Extract<LexicalOutcomeMatch, { kind: 'exact' }>,
|
|
1235
|
+
candidate: Extract<LexicalOutcomeMatch, { kind: 'startsWith' | 'endsWith' }>,
|
|
1236
|
+
): boolean => {
|
|
1237
|
+
if (exact.caseInsensitive && !candidate.caseInsensitive && /[A-Za-z]/.test(candidate.value)) return false
|
|
1238
|
+
const needle = candidate.caseInsensitive ? foldAscii(candidate.value) : candidate.value
|
|
1239
|
+
return exact.values.every(raw => {
|
|
1240
|
+
const value = candidate.caseInsensitive ? foldAscii(raw) : raw
|
|
1241
|
+
return candidate.kind === 'startsWith' ? value.startsWith(needle) : value.endsWith(needle)
|
|
1242
|
+
})
|
|
1243
|
+
}
|
|
1244
|
+
const reduced: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[] = []
|
|
1245
|
+
for (const entry of excluding) {
|
|
1246
|
+
if (entry.kind !== 'exact') { reduced.push(entry); continue }
|
|
1247
|
+
const values = entry.values.filter(value => !fixed.some(candidate =>
|
|
1248
|
+
fixedContainsExact({ ...entry, values: [value] }, candidate)))
|
|
1249
|
+
if (values.length > 0) reduced.push({ ...entry, values })
|
|
1250
|
+
}
|
|
1251
|
+
return JSON.stringify({
|
|
1252
|
+
kind: match.kind,
|
|
1253
|
+
excluding: [...new Set(reduced.map(canonicalLexicalOutcomeKey))].sort(),
|
|
1254
|
+
})
|
|
1255
|
+
}
|
|
1256
|
+
|
|
45
1257
|
function keyOf(def: ParserDef): string | undefined {
|
|
46
1258
|
switch (def.tag) {
|
|
47
1259
|
case 'literal': return `L\u0000${def.value}\u0000${def.caseInsensitive ? 'i' : ''}`
|
|
@@ -60,6 +1272,44 @@ function terminalOf(def: ParserDef, id: number): TokenTerminal | undefined {
|
|
|
60
1272
|
}
|
|
61
1273
|
}
|
|
62
1274
|
|
|
1275
|
+
/** Does a final rule-map winner bottom out at the reference being resolved? */
|
|
1276
|
+
export function winnerWrapsReference(
|
|
1277
|
+
winner: Combinator<unknown>,
|
|
1278
|
+
reference: Combinator<unknown>,
|
|
1279
|
+
): boolean {
|
|
1280
|
+
let current = winner
|
|
1281
|
+
const seen = new Set<Combinator<unknown>>()
|
|
1282
|
+
// Only wrappers introduced by rules()/parser() are transparent here. Walking
|
|
1283
|
+
// into a sequence or choice would mistake ordinary recursive use for an alias.
|
|
1284
|
+
while (!seen.has(current)) {
|
|
1285
|
+
if (current === reference) return true
|
|
1286
|
+
seen.add(current)
|
|
1287
|
+
const def = current._def
|
|
1288
|
+
if (def.tag !== 'grammar' && def.tag !== 'trivia') return false
|
|
1289
|
+
current = def.parser
|
|
1290
|
+
}
|
|
1291
|
+
return false
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
/** Final winner resolution is authoritative; a stale construction thunk is fallback only. */
|
|
1295
|
+
function resolvedLazyTarget(
|
|
1296
|
+
parser: Combinator<unknown>,
|
|
1297
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
1298
|
+
): Combinator<unknown> | undefined {
|
|
1299
|
+
const def = parser._def
|
|
1300
|
+
if (def.tag !== 'lazy') return undefined
|
|
1301
|
+
const name = (parser as unknown as { _ruleName?: string })._ruleName
|
|
1302
|
+
if (name !== undefined && resolve !== undefined) {
|
|
1303
|
+
const winner = resolve(name)
|
|
1304
|
+
// A named rule whose whole definition is itself a lazy alias is stamped
|
|
1305
|
+
// with its OWN name. Resolving that name returns the same object, not the
|
|
1306
|
+
// referenced winner; only a distinct winner supersedes the construction
|
|
1307
|
+
// thunk.
|
|
1308
|
+
if (winner !== undefined && !winnerWrapsReference(winner, parser)) return winner
|
|
1309
|
+
}
|
|
1310
|
+
try { return def.thunk() } catch { return undefined }
|
|
1311
|
+
}
|
|
1312
|
+
|
|
63
1313
|
/** Direct sub-parsers of a def, resolving a lazy through `resolve` when its own thunk is a hole. */
|
|
64
1314
|
export function tokenChildren(
|
|
65
1315
|
p: Combinator<unknown>,
|
|
@@ -78,13 +1328,7 @@ export function tokenChildren(
|
|
|
78
1328
|
if (Array.isArray(rec.cases)) for (const c of rec.cases as Array<{ parser: Combinator<unknown> }>) push(c.parser)
|
|
79
1329
|
if (Array.isArray(rec.matchers)) for (const c of rec.matchers as Array<{ parser: Combinator<unknown> }>) push(c.parser)
|
|
80
1330
|
if (d.tag === 'lazy') {
|
|
81
|
-
|
|
82
|
-
try { target = d.thunk() } catch { target = undefined }
|
|
83
|
-
if (target === undefined && resolve !== undefined) {
|
|
84
|
-
const name = (p as unknown as { _ruleName?: string })._ruleName
|
|
85
|
-
if (name !== undefined) target = resolve(name)
|
|
86
|
-
}
|
|
87
|
-
push(target)
|
|
1331
|
+
push(resolvedLazyTarget(p, resolve))
|
|
88
1332
|
}
|
|
89
1333
|
return out
|
|
90
1334
|
}
|
|
@@ -121,6 +1365,3066 @@ export function collectAlphabet(
|
|
|
121
1365
|
return { terminals, byKey, originOf }
|
|
122
1366
|
}
|
|
123
1367
|
|
|
1368
|
+
type Normalized = { ir: LexicalIr } | { refusal: string }
|
|
1369
|
+
|
|
1370
|
+
type PendingLexicalWrapperFrame =
|
|
1371
|
+
| {
|
|
1372
|
+
readonly id: number
|
|
1373
|
+
readonly parentFrameId?: number
|
|
1374
|
+
readonly controlId: number
|
|
1375
|
+
readonly kind: 'token'
|
|
1376
|
+
stateStart: number
|
|
1377
|
+
stateEnd: number
|
|
1378
|
+
}
|
|
1379
|
+
| {
|
|
1380
|
+
readonly id: number
|
|
1381
|
+
readonly parentFrameId?: number
|
|
1382
|
+
readonly controlId: number
|
|
1383
|
+
readonly kind: 'grammar'
|
|
1384
|
+
readonly parser: Combinator<unknown>
|
|
1385
|
+
stateStart: number
|
|
1386
|
+
stateEnd: number
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
type LexicalNormalizeSession = {
|
|
1390
|
+
nextState: number
|
|
1391
|
+
readonly expected: string[][]
|
|
1392
|
+
readonly expectedIds: Map<string, number>
|
|
1393
|
+
readonly events: LexicalDiagnosticEvent[]
|
|
1394
|
+
readonly wrapperFrames: PendingLexicalWrapperFrame[]
|
|
1395
|
+
readonly wrapperStack: number[]
|
|
1396
|
+
readonly controls: LexicalControlNode[]
|
|
1397
|
+
readonly controlStack: number[]
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
function newNormalizeSession(): LexicalNormalizeSession {
|
|
1401
|
+
return {
|
|
1402
|
+
nextState: 0, expected: [], expectedIds: new Map(), events: [],
|
|
1403
|
+
wrapperFrames: [], wrapperStack: [], controls: [], controlStack: [],
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
function withLexicalControl(
|
|
1408
|
+
session: LexicalNormalizeSession | undefined,
|
|
1409
|
+
kind: LexicalControlNode['kind'],
|
|
1410
|
+
body: () => Normalized,
|
|
1411
|
+
): Normalized {
|
|
1412
|
+
if (session === undefined) return body()
|
|
1413
|
+
const id = session.controls.length
|
|
1414
|
+
const parentControlId = session.controlStack[session.controlStack.length - 1]
|
|
1415
|
+
const node: LexicalControlNode = {
|
|
1416
|
+
id, ...(parentControlId === undefined ? {} : { parentControlId }), kind,
|
|
1417
|
+
stateStart: session.nextState, stateEnd: -1,
|
|
1418
|
+
}
|
|
1419
|
+
session.controls.push(node)
|
|
1420
|
+
session.controlStack.push(id)
|
|
1421
|
+
try {
|
|
1422
|
+
const result = body()
|
|
1423
|
+
;(node as { stateEnd: number }).stateEnd = session.nextState
|
|
1424
|
+
return result
|
|
1425
|
+
} finally {
|
|
1426
|
+
const popped = session.controlStack.pop()
|
|
1427
|
+
if (popped !== id) throw new Error('parseman: lexical control stack is not well nested')
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
function withLexicalWrapper(
|
|
1432
|
+
session: LexicalNormalizeSession | undefined,
|
|
1433
|
+
kind: 'token' | 'grammar',
|
|
1434
|
+
parser: Combinator<unknown>,
|
|
1435
|
+
body: () => Normalized,
|
|
1436
|
+
): Normalized {
|
|
1437
|
+
if (session === undefined) return body()
|
|
1438
|
+
const controlId = session.controlStack[session.controlStack.length - 1]
|
|
1439
|
+
if (controlId === undefined) throw new Error('parseman: lexical wrapper lacks a control anchor')
|
|
1440
|
+
const id = session.wrapperFrames.length
|
|
1441
|
+
const parentFrameId = session.wrapperStack[session.wrapperStack.length - 1]
|
|
1442
|
+
const common = {
|
|
1443
|
+
id, ...(parentFrameId === undefined ? {} : { parentFrameId }), controlId,
|
|
1444
|
+
stateStart: session.nextState, stateEnd: -1,
|
|
1445
|
+
}
|
|
1446
|
+
const frame: PendingLexicalWrapperFrame = kind === 'token'
|
|
1447
|
+
? { ...common, kind }
|
|
1448
|
+
: { ...common, kind, parser }
|
|
1449
|
+
session.wrapperFrames.push(frame)
|
|
1450
|
+
session.wrapperStack.push(id)
|
|
1451
|
+
try {
|
|
1452
|
+
const result = body()
|
|
1453
|
+
frame.stateEnd = session.nextState
|
|
1454
|
+
return result
|
|
1455
|
+
} finally {
|
|
1456
|
+
const popped = session.wrapperStack.pop()
|
|
1457
|
+
if (popped !== id) throw new Error('parseman: lexical wrapper stack is not well nested')
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
function lexicalState(session?: LexicalNormalizeSession): number {
|
|
1462
|
+
return session === undefined ? -1 : session.nextState++
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
function lexicalExpected(session: LexicalNormalizeSession, expected: readonly string[]): number {
|
|
1466
|
+
const key = JSON.stringify(expected)
|
|
1467
|
+
const prior = session.expectedIds.get(key)
|
|
1468
|
+
if (prior !== undefined) return prior
|
|
1469
|
+
const id = session.expected.length
|
|
1470
|
+
session.expectedIds.set(key, id)
|
|
1471
|
+
session.expected.push([...expected])
|
|
1472
|
+
return id
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
function lexicalFail(session: LexicalNormalizeSession | undefined, state: number, expected: readonly string[]): void {
|
|
1476
|
+
if (session !== undefined) {
|
|
1477
|
+
session.events.push({ op: 'FAIL', state, expectedId: lexicalExpected(session, expected) })
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
function regexFlags(flags: string): string {
|
|
1482
|
+
// regex() itself owns stickiness and ignores authored global/sticky state.
|
|
1483
|
+
return new RegExp('', flags.replace(/[gy]/g, '')).flags
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
function sequenceIr(parts: LexicalIr[]): LexicalIr {
|
|
1487
|
+
const flat: LexicalIr[] = []
|
|
1488
|
+
for (const part of parts) {
|
|
1489
|
+
if (part.kind === 'sequence') flat.push(...part.parts)
|
|
1490
|
+
else flat.push(part)
|
|
1491
|
+
}
|
|
1492
|
+
return flat.length === 1 ? flat[0]! : { kind: 'sequence', parts: flat }
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
/**
|
|
1496
|
+
* The first production normalization is deliberately proof-bounded: a single
|
|
1497
|
+
* repeated positive character class followed by one optional escaped literal.
|
|
1498
|
+
* This covers the identifier + optional '(' family while keeping regex
|
|
1499
|
+
* backtracking distinct from PEG possessive repetition everywhere else.
|
|
1500
|
+
*/
|
|
1501
|
+
function provenOptionalLiteralSuffix(
|
|
1502
|
+
source: string,
|
|
1503
|
+
flags: string,
|
|
1504
|
+
): { base: string; suffix: string } | undefined {
|
|
1505
|
+
const match = /^(\[(?:\\.|[^\]\\])+\]\+)(\\[.*+?^$(){}|[\]\\/])\?$/.exec(source)
|
|
1506
|
+
if (match === null) return undefined
|
|
1507
|
+
const base = match[1]!
|
|
1508
|
+
const suffix = match[2]!.slice(1)
|
|
1509
|
+
// A one-character oracle is exact for this bounded base shape: the base is one
|
|
1510
|
+
// positive character class repeated with '+'. If the class cannot consume the
|
|
1511
|
+
// suffix character, the regex quantifier cannot backtrack across the suffix.
|
|
1512
|
+
const baseFlags = regexFlags(flags)
|
|
1513
|
+
if (new RegExp(`^(?:${base})$`, baseFlags).test(suffix)) return undefined
|
|
1514
|
+
return { base, suffix }
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
function normalizeRegex(source: string, flags: string, session?: LexicalNormalizeSession): LexicalIr {
|
|
1518
|
+
const failure = directTerminalFailureExpected({ tag: 'regex', source, flags })
|
|
1519
|
+
const proof = provenOptionalLiteralSuffix(source, flags)
|
|
1520
|
+
if (proof === undefined) {
|
|
1521
|
+
const state = lexicalState(session)
|
|
1522
|
+
lexicalFail(session, state, failure)
|
|
1523
|
+
return { kind: 'regex', source, flags }
|
|
1524
|
+
}
|
|
1525
|
+
lexicalState(session) // synthetic sequence
|
|
1526
|
+
const baseState = lexicalState(session)
|
|
1527
|
+
lexicalFail(session, baseState, failure)
|
|
1528
|
+
lexicalState(session) // synthetic optional repeat
|
|
1529
|
+
lexicalState(session) // synthetic optional literal; no invented FAIL event
|
|
1530
|
+
return {
|
|
1531
|
+
kind: 'sequence',
|
|
1532
|
+
parts: [
|
|
1533
|
+
{ kind: 'regex', source: proof.base, flags },
|
|
1534
|
+
{
|
|
1535
|
+
kind: 'repeat',
|
|
1536
|
+
body: {
|
|
1537
|
+
kind: 'literal',
|
|
1538
|
+
value: proof.suffix,
|
|
1539
|
+
caseInsensitive: flags.includes('i') && /[A-Za-z]/.test(proof.suffix),
|
|
1540
|
+
},
|
|
1541
|
+
min: 0,
|
|
1542
|
+
max: 1,
|
|
1543
|
+
greedy: true,
|
|
1544
|
+
mode: 'possessive',
|
|
1545
|
+
},
|
|
1546
|
+
],
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
function normalizeLexical(
|
|
1551
|
+
parser: Combinator<unknown>,
|
|
1552
|
+
resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
|
|
1553
|
+
stack: Set<Combinator<unknown>>,
|
|
1554
|
+
session?: LexicalNormalizeSession,
|
|
1555
|
+
): Normalized {
|
|
1556
|
+
if (stack.has(parser)) return { refusal: 'recursive token body' }
|
|
1557
|
+
stack.add(parser)
|
|
1558
|
+
try {
|
|
1559
|
+
return withLexicalControl(session, parser._def.tag, () =>
|
|
1560
|
+
normalizeLexicalBody(parser, resolve, stack, session))
|
|
1561
|
+
} finally {
|
|
1562
|
+
stack.delete(parser)
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
function normalizeLexicalBody(
|
|
1567
|
+
parser: Combinator<unknown>,
|
|
1568
|
+
resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
|
|
1569
|
+
stack: Set<Combinator<unknown>>,
|
|
1570
|
+
session?: LexicalNormalizeSession,
|
|
1571
|
+
): Normalized {
|
|
1572
|
+
const done = (result: Normalized): Normalized => result
|
|
1573
|
+
const child = (value: Combinator<unknown>): Normalized => normalizeLexical(value, resolve, stack, session)
|
|
1574
|
+
const def = parser._def
|
|
1575
|
+
|
|
1576
|
+
switch (def.tag) {
|
|
1577
|
+
case 'literal': {
|
|
1578
|
+
const state = lexicalState(session)
|
|
1579
|
+
lexicalFail(session, state, directTerminalFailureExpected(def))
|
|
1580
|
+
return done({ ir: {
|
|
1581
|
+
kind: 'literal',
|
|
1582
|
+
value: def.caseInsensitive ? def.value.replace(/[A-Z]/g, c => c.toLowerCase()) : def.value,
|
|
1583
|
+
caseInsensitive: def.caseInsensitive,
|
|
1584
|
+
} })
|
|
1585
|
+
}
|
|
1586
|
+
case 'keywords': {
|
|
1587
|
+
const state = lexicalState(session)
|
|
1588
|
+
lexicalFail(session, state, directTerminalFailureExpected(def))
|
|
1589
|
+
const words = [...new Set(def.words.map(word => def.caseInsensitive
|
|
1590
|
+
? word.replace(/[A-Z]/g, c => c.toLowerCase())
|
|
1591
|
+
: word))].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0))
|
|
1592
|
+
return done({ ir: {
|
|
1593
|
+
kind: 'keywords', words, caseInsensitive: def.caseInsensitive, boundary: def.boundary,
|
|
1594
|
+
} })
|
|
1595
|
+
}
|
|
1596
|
+
case 'regex':
|
|
1597
|
+
return done({ ir: normalizeRegex(def.source, regexFlags(def.flags), session) })
|
|
1598
|
+
case 'sequence': {
|
|
1599
|
+
lexicalState(session)
|
|
1600
|
+
const parts: LexicalIr[] = []
|
|
1601
|
+
for (const entry of def.parsers) {
|
|
1602
|
+
const normalized = child(entry)
|
|
1603
|
+
if ('refusal' in normalized) return done(normalized)
|
|
1604
|
+
if (normalized.ir.kind === 'sequence') parts.push(...normalized.ir.parts)
|
|
1605
|
+
else parts.push(normalized.ir)
|
|
1606
|
+
}
|
|
1607
|
+
return done({ ir: sequenceIr(parts) })
|
|
1608
|
+
}
|
|
1609
|
+
case 'choice': {
|
|
1610
|
+
lexicalState(session)
|
|
1611
|
+
if (def.gates.some(gate => gate !== null)) return done({ refusal: 'choice has a dynamic gate' })
|
|
1612
|
+
if (def.autoNot.some(check => check !== null)) return done({ refusal: 'choice has contextual prefix rejection' })
|
|
1613
|
+
if (!def.disjoint && def.strategy.tag !== 'firstMatch') {
|
|
1614
|
+
return done({ refusal: `choice strategy ${def.strategy.tag} is not canonical lexical order` })
|
|
1615
|
+
}
|
|
1616
|
+
const arms: LexicalIr[] = []
|
|
1617
|
+
for (const entry of def.parsers) {
|
|
1618
|
+
const normalized = child(entry)
|
|
1619
|
+
if ('refusal' in normalized) return done(normalized)
|
|
1620
|
+
arms.push(normalized.ir)
|
|
1621
|
+
}
|
|
1622
|
+
return done({ ir: arms.length === 1 ? arms[0]! : { kind: 'choice', arms } })
|
|
1623
|
+
}
|
|
1624
|
+
case 'optional': {
|
|
1625
|
+
lexicalState(session)
|
|
1626
|
+
const normalized = child(def.parser)
|
|
1627
|
+
return done('refusal' in normalized ? normalized : {
|
|
1628
|
+
ir: { kind: 'repeat', body: normalized.ir, min: 0, max: 1, greedy: true, mode: 'possessive' },
|
|
1629
|
+
})
|
|
1630
|
+
}
|
|
1631
|
+
case 'many':
|
|
1632
|
+
case 'oneOrMore': {
|
|
1633
|
+
const state = lexicalState(session)
|
|
1634
|
+
if (matchesEmpty(def.parser)) return done({ refusal: 'repeat body may not make progress' })
|
|
1635
|
+
const childStart = session?.nextState ?? -1
|
|
1636
|
+
const normalized = child(def.parser)
|
|
1637
|
+
if ('refusal' in normalized) return done(normalized)
|
|
1638
|
+
if (session !== undefined && def.min > 1) {
|
|
1639
|
+
const expected = deriveExpected(def.parser)
|
|
1640
|
+
session.events.push({
|
|
1641
|
+
op: 'REQUIRE', state, childStart, childEnd: session.nextState,
|
|
1642
|
+
expectedId: lexicalExpected(session, expected.length > 0 ? expected : [def.parser._tag]),
|
|
1643
|
+
when: 'mandatory-iteration-2-to-min', committedChild: 'propagate',
|
|
1644
|
+
probe: 'child-only', anchor: 'repeat-position',
|
|
1645
|
+
})
|
|
1646
|
+
}
|
|
1647
|
+
return done({
|
|
1648
|
+
ir: { kind: 'repeat', body: normalized.ir, min: def.min, max: def.max ?? null, greedy: true, mode: 'possessive' },
|
|
1649
|
+
})
|
|
1650
|
+
}
|
|
1651
|
+
case 'not':
|
|
1652
|
+
case 'peek': {
|
|
1653
|
+
const state = lexicalState(session)
|
|
1654
|
+
const innerStart = session?.nextState ?? -1
|
|
1655
|
+
const normalized = child(def.parser)
|
|
1656
|
+
if ('refusal' in normalized) return done(normalized)
|
|
1657
|
+
if (session !== undefined) session.events.push({
|
|
1658
|
+
op: 'ASSERT', state, innerStart, innerEnd: session.nextState,
|
|
1659
|
+
positive: def.tag === 'peek',
|
|
1660
|
+
expectedId: lexicalExpected(session, assertionFailureExpected(def.tag === 'peek', def.parser._tag)),
|
|
1661
|
+
snapshotPolicy: 'saveLookaheadMark', execution: 'deferred',
|
|
1662
|
+
})
|
|
1663
|
+
return done({
|
|
1664
|
+
ir: { kind: 'assert', positive: def.tag === 'peek', body: normalized.ir },
|
|
1665
|
+
})
|
|
1666
|
+
}
|
|
1667
|
+
case 'lazy': {
|
|
1668
|
+
const target = resolvedLazyTarget(parser, resolve)
|
|
1669
|
+
return done(target === undefined ? { refusal: 'unresolved lazy token body' } : child(target))
|
|
1670
|
+
}
|
|
1671
|
+
case 'token':
|
|
1672
|
+
return done(withLexicalWrapper(session, 'token', parser, () => child(def.parser)))
|
|
1673
|
+
case 'trivia':
|
|
1674
|
+
return done(child(def.parser))
|
|
1675
|
+
case 'grammar':
|
|
1676
|
+
// A direct token child installs its own contiguous lexical context before
|
|
1677
|
+
// its body runs. The enclosing scope's trivia/capture/line policy is
|
|
1678
|
+
// therefore recognition-inert (its observable effects remain a separate
|
|
1679
|
+
// capability obligation). Keep every other trivia-bearing scope distinct:
|
|
1680
|
+
// trivia between ordinary child terms changes the accepted language.
|
|
1681
|
+
if (def.parser._def.tag === 'token') {
|
|
1682
|
+
return done(withLexicalWrapper(session, 'grammar', parser, () => child(def.parser)))
|
|
1683
|
+
}
|
|
1684
|
+
if (def.trackLines) return done({ refusal: 'token body has line-tracking effects' })
|
|
1685
|
+
if (def.triviaParser !== undefined || def.captureTrivia === true || def.rootCapture !== undefined) {
|
|
1686
|
+
return done({ refusal: 'token body has a trivia-bearing scope' })
|
|
1687
|
+
}
|
|
1688
|
+
return done(withLexicalWrapper(session, 'grammar', parser, () => child(def.parser)))
|
|
1689
|
+
case 'transform': return done({ refusal: 'transform is effectful' })
|
|
1690
|
+
case 'leaf': return done({ refusal: 'leaf is effectful' })
|
|
1691
|
+
case 'node': return done({ refusal: 'node is effectful' })
|
|
1692
|
+
case 'field': return done({ refusal: 'field is effectful' })
|
|
1693
|
+
case 'label': return done({ refusal: 'label changes diagnostics' })
|
|
1694
|
+
case 'expect': return done({ refusal: 'expect changes diagnostics' })
|
|
1695
|
+
case 'attempt': return done({ refusal: 'attempt changes commitment' })
|
|
1696
|
+
case 'withCtx': return done({ refusal: 'withCtx is dynamic' })
|
|
1697
|
+
case 'guard': return done({ refusal: 'guard is dynamic' })
|
|
1698
|
+
case 'adjacency': return done({ refusal: 'adjacency depends on parser state' })
|
|
1699
|
+
case 'recover': return done({ refusal: 'recover is effectful' })
|
|
1700
|
+
case 'scanTo': return done({ refusal: 'scanTo is not a lexical run' })
|
|
1701
|
+
case 'sepBy': return done({ refusal: 'separated repetition is not a lexical run' })
|
|
1702
|
+
case 'dispatch': return done({ refusal: 'nested dispatch has semantic outcomes' })
|
|
1703
|
+
case 'routed': return done({ refusal: 'routed depends on an outer dispatch' })
|
|
1704
|
+
case 'unknown': return done({ refusal: 'unknown parser body' })
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
/**
|
|
1709
|
+
* `balanced()` deliberately overrides its eager structural `_def`; the
|
|
1710
|
+
* constructor marker is the only authoritative recognition language. Model it
|
|
1711
|
+
* directly instead of rediscovering a recursive transform/expect expansion.
|
|
1712
|
+
*/
|
|
1713
|
+
function normalizeBalancedLexical(
|
|
1714
|
+
parser: Combinator<unknown>,
|
|
1715
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
1716
|
+
session?: LexicalNormalizeSession,
|
|
1717
|
+
): Normalized | undefined {
|
|
1718
|
+
// `token(balanced(...))` is legal and appears in Jess. The outer token owns
|
|
1719
|
+
// the public leaf, while the inner balanced token owns the constructor marker.
|
|
1720
|
+
let marked = parser
|
|
1721
|
+
let spec = (marked as BalancedSpec)._balancedSpec
|
|
1722
|
+
const tokenWrappers: Combinator<unknown>[] = []
|
|
1723
|
+
while (marked._def.tag === 'token') {
|
|
1724
|
+
tokenWrappers.push(marked)
|
|
1725
|
+
marked = marked._def.parser
|
|
1726
|
+
spec ??= (marked as BalancedSpec)._balancedSpec
|
|
1727
|
+
}
|
|
1728
|
+
if (spec === undefined) return undefined
|
|
1729
|
+
const machine = (): Normalized => withLexicalControl(session, 'balanced', () => {
|
|
1730
|
+
lexicalState(session) // balanced machine
|
|
1731
|
+
const openState = lexicalState(session)
|
|
1732
|
+
if (session !== undefined) lexicalFail(session, openState, directTerminalFailureExpected({
|
|
1733
|
+
tag: 'literal', value: spec.open, caseInsensitive: false,
|
|
1734
|
+
}))
|
|
1735
|
+
lexicalState(session) // balanced body scan
|
|
1736
|
+
const skip: LexicalIr[] = []
|
|
1737
|
+
// Ambient scanSkip is an observable runtime callback policy: balanced keys
|
|
1738
|
+
// its rebuilt interior by the live array identity and dynamically looks up
|
|
1739
|
+
// every callback's `.parse`. Only authored ownSkip belongs to canonical IR.
|
|
1740
|
+
for (const entry of spec.ownSkip) {
|
|
1741
|
+
const normalized = withLexicalControl(session, 'balanced-skip', () =>
|
|
1742
|
+
normalizeBalancedLexical(entry, resolve, session)
|
|
1743
|
+
?? normalizeLexical(entry, resolve, new Set(), session))
|
|
1744
|
+
if ('refusal' in normalized) return {
|
|
1745
|
+
refusal: `balanced skipper is not represented: ${normalized.refusal}`,
|
|
1746
|
+
}
|
|
1747
|
+
skip.push(normalized.ir)
|
|
1748
|
+
}
|
|
1749
|
+
const closeState = lexicalState(session)
|
|
1750
|
+
if (session !== undefined) {
|
|
1751
|
+
const expectedId = lexicalExpected(session, directTerminalFailureExpected({
|
|
1752
|
+
tag: 'literal', value: spec.close, caseInsensitive: false,
|
|
1753
|
+
}))
|
|
1754
|
+
session.events.push(spec.strict
|
|
1755
|
+
? {
|
|
1756
|
+
op: 'BAL_CLOSE_STRICT', state: closeState, expectedId, probe: true, committed: false,
|
|
1757
|
+
when: 'close-miss-after-open-body-success',
|
|
1758
|
+
}
|
|
1759
|
+
: {
|
|
1760
|
+
op: 'BAL_CLOSE_RECOVER', state: closeState, expectedId,
|
|
1761
|
+
probe: true, committed: false, when: 'close-miss-after-open-body-success',
|
|
1762
|
+
result: 'parseError', errorSpan: 'close-position',
|
|
1763
|
+
lineAnnotation: 'when-active', errorSink: 'when-active', cstErrorCapture: 'when-active',
|
|
1764
|
+
})
|
|
1765
|
+
}
|
|
1766
|
+
return {
|
|
1767
|
+
ir: {
|
|
1768
|
+
kind: 'balanced', open: spec.open, close: spec.close,
|
|
1769
|
+
strict: spec.strict, raw: spec.raw, skip,
|
|
1770
|
+
},
|
|
1771
|
+
}
|
|
1772
|
+
})
|
|
1773
|
+
const wrap = (index: number): Normalized => {
|
|
1774
|
+
const tokenParser = tokenWrappers[index]
|
|
1775
|
+
return tokenParser === undefined ? machine()
|
|
1776
|
+
: withLexicalControl(session, 'token', () =>
|
|
1777
|
+
withLexicalWrapper(session, 'token', tokenParser, () => wrap(index + 1)))
|
|
1778
|
+
}
|
|
1779
|
+
return wrap(0)
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
const CAPABILITY_VARIANT_GAP =
|
|
1783
|
+
'total token replacement body is not implemented for every supported assembly variant'
|
|
1784
|
+
const FIXED_TUPLE_BINDING_GAP =
|
|
1785
|
+
'binding candidate set is incomplete: shared, direct captured, and statically named fixed-tuple bodies have not all been built before pricing; fixed children and routes may not be rediscovered from arrays'
|
|
1786
|
+
const COMPLETE_CAPABILITY = { kind: 'complete' } as const
|
|
1787
|
+
const gap = (reason: string): LexicalCapabilityStatus => ({ kind: 'gap', reason })
|
|
1788
|
+
const COMPLETE_PHASE: LexicalCapabilityPhase = {
|
|
1789
|
+
representation: COMPLETE_CAPABILITY,
|
|
1790
|
+
executableLowering: COMPLETE_CAPABILITY,
|
|
1791
|
+
}
|
|
1792
|
+
const phaseGap = (representation: LexicalCapabilityStatus, loweringReason: string): LexicalCapabilityPhase => ({
|
|
1793
|
+
representation,
|
|
1794
|
+
executableLowering: representation.kind === 'impossible'
|
|
1795
|
+
? representation : gap(loweringReason),
|
|
1796
|
+
})
|
|
1797
|
+
|
|
1798
|
+
const ALL_BINDING_READERS = 0b111
|
|
1799
|
+
const ALL_BINDING_VARIANTS = 0b11_1111
|
|
1800
|
+
const TERMINAL_CAPTURED_TEMPLATE_ID = 11
|
|
1801
|
+
const DECISION_REFERENCE_TEMPLATE_ID = 12
|
|
1802
|
+
const DECISION_CAPTURED_TEMPLATE_ID = 13
|
|
1803
|
+
const DECISION_NAMED_TEMPLATE_ID = 14
|
|
1804
|
+
|
|
1805
|
+
/** Stable pre-pricing template ids. They name existing reader constructions;
|
|
1806
|
+
* they are deliberately not opcodes and never enter the parse path. */
|
|
1807
|
+
function bindingTemplateId(parentTag: ParserDef['tag'] | 'root', ordinal: number): number {
|
|
1808
|
+
if (parentTag === 'root') return 1
|
|
1809
|
+
switch (parentTag) {
|
|
1810
|
+
case 'sequence': return 2
|
|
1811
|
+
case 'choice': return 3
|
|
1812
|
+
case 'dispatch': return ordinal === 0 ? 4 : 5
|
|
1813
|
+
case 'many': case 'oneOrMore': case 'sepBy': return ordinal === 0 ? 6 : 7
|
|
1814
|
+
case 'scanTo': return ordinal === 0 ? 8 : 9
|
|
1815
|
+
default: return 10
|
|
1816
|
+
}
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
function bindingDigest(words: readonly number[], parentTag: string, childTag: string): number {
|
|
1820
|
+
let hash = 0x811c9dc5
|
|
1821
|
+
const feed = (value: number): void => { hash ^= value & 0xFF; hash = Math.imul(hash, 0x01000193) }
|
|
1822
|
+
for (const c of `${parentTag}\u0000${childTag}`) feed(c.charCodeAt(0))
|
|
1823
|
+
for (const word of words) {
|
|
1824
|
+
feed(word); feed(word >>> 8); feed(word >>> 16); feed(word >>> 24)
|
|
1825
|
+
}
|
|
1826
|
+
return hash >>> 0
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
function fixedBindingProjection(
|
|
1830
|
+
edge: BindingEdgeCandidate,
|
|
1831
|
+
edgeId: number,
|
|
1832
|
+
projectionId: number,
|
|
1833
|
+
): LexicalBindingProjection {
|
|
1834
|
+
const template = bindingTemplateId(edge.parentTag, edge.childOrdinal)
|
|
1835
|
+
let readerMask = ALL_BINDING_READERS
|
|
1836
|
+
const variantMask = ALL_BINDING_VARIANTS
|
|
1837
|
+
// Scanner children already have one fixed projection per reader: execRules
|
|
1838
|
+
// uses their numeric SubtreeRef, the closure adapter captures `pieceAt(ip)`,
|
|
1839
|
+
// and the emitted adapter captures that site's named body after linking every
|
|
1840
|
+
// scan root. Only ambient trivia/scanSkip remain live runtime callbacks; they
|
|
1841
|
+
// are scanner POLICY, never binding authority for these authored own children.
|
|
1842
|
+
// Recovery/withCtx still have no equivalent named template.
|
|
1843
|
+
if (edge.parentTag === 'recover' || edge.parentTag === 'withCtx') readerMask = 0b011
|
|
1844
|
+
// Sequence strict, adjacency, and recovery bodies now all use scalar block
|
|
1845
|
+
// chains, so no supported variant is withheld here.
|
|
1846
|
+
const words = [
|
|
1847
|
+
edge.parentBodyId, edge.childBodyId, edge.childOrdinal,
|
|
1848
|
+
edge.childOrdinal, edge.childOrdinal, template,
|
|
1849
|
+
edge.childBodyId, template, readerMask, variantMask,
|
|
1850
|
+
]
|
|
1851
|
+
return {
|
|
1852
|
+
id: projectionId, edgeId,
|
|
1853
|
+
parentBodyId: edge.parentBodyId, childBodyId: edge.childBodyId,
|
|
1854
|
+
childOrdinal: edge.childOrdinal, referenceOperand: edge.childOrdinal,
|
|
1855
|
+
capturedSlotId: edge.childOrdinal, capturedTemplateId: template,
|
|
1856
|
+
namedSymbolId: edge.childBodyId, namedTemplateId: template,
|
|
1857
|
+
readerMask, variantMask,
|
|
1858
|
+
semanticDigest: bindingDigest(words, edge.parentTag, edge.childTag),
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
function derivedCapabilityStatus(obligations: LexicalCapabilityObligations): LexicalCapabilityStatus {
|
|
1863
|
+
const phases = [
|
|
1864
|
+
obligations.recognition,
|
|
1865
|
+
obligations.diagnostics,
|
|
1866
|
+
obligations.boundaryPlan,
|
|
1867
|
+
obligations.materializationPlan,
|
|
1868
|
+
obligations.supportedVariants,
|
|
1869
|
+
obligations.bindingAndReachability,
|
|
1870
|
+
]
|
|
1871
|
+
const ordered = phases.flatMap(entry => [entry.representation, entry.executableLowering])
|
|
1872
|
+
const impossible = ordered.find((entry): entry is Extract<LexicalCapabilityStatus, { kind: 'impossible' }> =>
|
|
1873
|
+
entry.kind === 'impossible')
|
|
1874
|
+
if (impossible !== undefined) return impossible
|
|
1875
|
+
// Capability is availability before selection: only executable lowering can
|
|
1876
|
+
// close it. Actual execution is test evidence, not a third status dimension.
|
|
1877
|
+
const gaps = phases.map(entry => entry.executableLowering)
|
|
1878
|
+
.filter((entry): entry is Extract<LexicalCapabilityStatus, { kind: 'gap' }> =>
|
|
1879
|
+
entry.kind === 'gap')
|
|
1880
|
+
if (gaps.length > 0) return gap([...new Set(gaps.map(entry => entry.reason))].join('; '))
|
|
1881
|
+
return COMPLETE_CAPABILITY
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
function tokenObligations(
|
|
1885
|
+
recognition: LexicalCapabilityStatus,
|
|
1886
|
+
represented: {
|
|
1887
|
+
readonly diagnostics: boolean
|
|
1888
|
+
readonly boundaryPlan: LexicalCapabilityStatus
|
|
1889
|
+
readonly materializationPlan: LexicalCapabilityStatus
|
|
1890
|
+
} = {
|
|
1891
|
+
diagnostics: false,
|
|
1892
|
+
boundaryPlan: gap('token boundary transaction is not represented'),
|
|
1893
|
+
materializationPlan: gap('token range/value/CST materialization is not represented'),
|
|
1894
|
+
},
|
|
1895
|
+
): LexicalCapabilityObligations {
|
|
1896
|
+
const representedOrGap = (yes: boolean, reason: string): LexicalCapabilityStatus =>
|
|
1897
|
+
yes ? COMPLETE_CAPABILITY : gap(reason)
|
|
1898
|
+
return {
|
|
1899
|
+
recognition: phaseGap(recognition, 'token recognizer lowering is not implemented for every supported variant'),
|
|
1900
|
+
diagnostics: phaseGap(
|
|
1901
|
+
representedOrGap(represented.diagnostics, 'token diagnostic plan is not represented'),
|
|
1902
|
+
'token diagnostic event lowering is deferred'),
|
|
1903
|
+
boundaryPlan: phaseGap(
|
|
1904
|
+
represented.boundaryPlan,
|
|
1905
|
+
'token boundary transaction lowering is deferred'),
|
|
1906
|
+
materializationPlan: phaseGap(
|
|
1907
|
+
represented.materializationPlan,
|
|
1908
|
+
'token range/value/CST materialization lowering is deferred'),
|
|
1909
|
+
supportedVariants: phaseGap(gap(CAPABILITY_VARIANT_GAP), CAPABILITY_VARIANT_GAP),
|
|
1910
|
+
bindingAndReachability: phaseGap(gap(FIXED_TUPLE_BINDING_GAP), FIXED_TUPLE_BINDING_GAP),
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
function executableLexBodyObligations(): LexicalCapabilityObligations {
|
|
1915
|
+
return {
|
|
1916
|
+
recognition: COMPLETE_PHASE,
|
|
1917
|
+
diagnostics: COMPLETE_PHASE,
|
|
1918
|
+
boundaryPlan: COMPLETE_PHASE,
|
|
1919
|
+
materializationPlan: COMPLETE_PHASE,
|
|
1920
|
+
supportedVariants: COMPLETE_PHASE,
|
|
1921
|
+
bindingAndReachability: COMPLETE_PHASE,
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
function terminalObligations(): LexicalCapabilityObligations {
|
|
1926
|
+
return {
|
|
1927
|
+
recognition: COMPLETE_PHASE,
|
|
1928
|
+
diagnostics: COMPLETE_PHASE,
|
|
1929
|
+
boundaryPlan: COMPLETE_PHASE,
|
|
1930
|
+
materializationPlan: COMPLETE_PHASE,
|
|
1931
|
+
supportedVariants: COMPLETE_PHASE,
|
|
1932
|
+
bindingAndReachability: phaseGap(gap(FIXED_TUPLE_BINDING_GAP), FIXED_TUPLE_BINDING_GAP),
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
function decisionObligations(recognition: LexicalCapabilityStatus): LexicalCapabilityObligations {
|
|
1937
|
+
return tokenObligations(recognition)
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
/** A final decision does not own a token boundary or token materialization.
|
|
1941
|
+
* Its executable projection is the existing canonical TableProgram row plus
|
|
1942
|
+
* the already-validated fixed child/effect projections. */
|
|
1943
|
+
function executableDecisionObligations(
|
|
1944
|
+
recognition: LexicalCapabilityStatus,
|
|
1945
|
+
effects: LexicalCapabilityPhase,
|
|
1946
|
+
): LexicalCapabilityObligations {
|
|
1947
|
+
const recognitionPhase = recognition.kind === 'complete'
|
|
1948
|
+
? COMPLETE_PHASE
|
|
1949
|
+
: phaseGap(recognition, 'decision recognition authority is incomplete')
|
|
1950
|
+
return {
|
|
1951
|
+
recognition: recognitionPhase,
|
|
1952
|
+
diagnostics: effects,
|
|
1953
|
+
boundaryPlan: COMPLETE_PHASE,
|
|
1954
|
+
materializationPlan: COMPLETE_PHASE,
|
|
1955
|
+
supportedVariants: effects,
|
|
1956
|
+
bindingAndReachability: effects,
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
|
|
1960
|
+
type DecisionLead = {
|
|
1961
|
+
readonly parser: Combinator<unknown>
|
|
1962
|
+
readonly ir: LexicalIr
|
|
1963
|
+
readonly key: string
|
|
1964
|
+
readonly dispatch?: Extract<ParserDef, { tag: 'dispatch' }>
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
type DecisionLeadResult = { readonly lead: DecisionLead } | { readonly gap: string }
|
|
1968
|
+
|
|
1969
|
+
/**
|
|
1970
|
+
* Find the one atomic range whose recognition starts this arm. This is only a
|
|
1971
|
+
* language proof: wrappers still owe diagnostics/effects and remain in the
|
|
1972
|
+
* eventual replacement body. Nullable prefixes and context-changing wrappers
|
|
1973
|
+
* deliberately decline the pruning proof rather than being guessed through.
|
|
1974
|
+
* The authored arm remains an explicit unrestricted TOKEN-body candidate; a
|
|
1975
|
+
* declined relation is not a missing recognizer and never licenses replay.
|
|
1976
|
+
*/
|
|
1977
|
+
function decisionLead(
|
|
1978
|
+
parser: Combinator<unknown>,
|
|
1979
|
+
context: CapabilityContext,
|
|
1980
|
+
resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
|
|
1981
|
+
seen = new Set<Combinator<unknown>>(),
|
|
1982
|
+
): DecisionLeadResult {
|
|
1983
|
+
if (seen.has(parser)) return { gap: 'recursive leading decision language' }
|
|
1984
|
+
seen.add(parser)
|
|
1985
|
+
const finish = (result: DecisionLeadResult): DecisionLeadResult => { seen.delete(parser); return result }
|
|
1986
|
+
const def = parser._def
|
|
1987
|
+
if (keyOf(def) !== undefined || def.tag === 'token' || (parser as BalancedSpec)._balancedSpec !== undefined) {
|
|
1988
|
+
const normalized = normalizeBalancedLexical(parser, resolve)
|
|
1989
|
+
?? normalizeLexical(def.tag === 'token' ? def.parser : parser, resolve, new Set())
|
|
1990
|
+
if ('refusal' in normalized) return finish({ gap: `leading recognizer: ${normalized.refusal}` })
|
|
1991
|
+
return finish({ lead: {
|
|
1992
|
+
parser,
|
|
1993
|
+
ir: normalized.ir,
|
|
1994
|
+
key: JSON.stringify(normalized.ir),
|
|
1995
|
+
} })
|
|
1996
|
+
}
|
|
1997
|
+
switch (def.tag) {
|
|
1998
|
+
case 'dispatch': {
|
|
1999
|
+
const selected = decisionLead(def.selector, context, resolve, seen)
|
|
2000
|
+
if ('gap' in selected) return finish(selected)
|
|
2001
|
+
return finish({ lead: { ...selected.lead, dispatch: def } })
|
|
2002
|
+
}
|
|
2003
|
+
case 'lazy': {
|
|
2004
|
+
const target = resolvedLazyTarget(parser, resolve)
|
|
2005
|
+
return finish(target === undefined
|
|
2006
|
+
? { gap: 'unresolved leading rule reference' }
|
|
2007
|
+
: decisionLead(target, context, resolve, seen))
|
|
2008
|
+
}
|
|
2009
|
+
case 'sequence':
|
|
2010
|
+
if (def.parsers.length === 0) return finish({ gap: 'empty leading sequence' })
|
|
2011
|
+
if (matchesEmpty(def.parsers[0]!, new Set(), resolve)) {
|
|
2012
|
+
return finish({ gap: 'nullable leading sequence needs a total same-position union proof' })
|
|
2013
|
+
}
|
|
2014
|
+
return finish(decisionLead(def.parsers[0]!, context, resolve, seen))
|
|
2015
|
+
case 'attempt': case 'transform': case 'leaf': case 'node': case 'field': case 'label': case 'expect':
|
|
2016
|
+
case 'trivia':
|
|
2017
|
+
return finish(decisionLead(def.parser, context, resolve, seen))
|
|
2018
|
+
case 'grammar':
|
|
2019
|
+
if (def.triviaParser !== undefined || def.clearTrivia === true || def.trackLines
|
|
2020
|
+
|| def.captureTrivia === true || def.rootCapture !== undefined) {
|
|
2021
|
+
return finish({ gap: 'leading grammar wrapper changes lexical context' })
|
|
2022
|
+
}
|
|
2023
|
+
return finish(decisionLead(def.parser, context, resolve, seen))
|
|
2024
|
+
case 'choice': return finish({ gap: 'nested leading choice needs a compatible range union proof' })
|
|
2025
|
+
case 'optional': case 'many': case 'oneOrMore': case 'sepBy':
|
|
2026
|
+
return finish({ gap: 'repeated or nullable leading language needs a range-partition proof' })
|
|
2027
|
+
case 'peek': return finish({ gap: 'positive lookahead needs an exact effect-free compatible view' })
|
|
2028
|
+
case 'not': return finish({ gap: 'negative lookahead needs a decidable complement' })
|
|
2029
|
+
case 'withCtx': case 'guard': case 'adjacency': case 'recover': case 'scanTo': case 'routed': case 'unknown':
|
|
2030
|
+
return finish({ gap: `leading ${def.tag} language is not represented` })
|
|
2031
|
+
case 'literal': case 'keywords': case 'regex':
|
|
2032
|
+
throw new Error('parseman: decision lead terminal escaped its direct case')
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
function finiteLexicalValues(ir: LexicalIr): readonly string[] | undefined {
|
|
2037
|
+
if (ir.kind === 'literal') return [ir.value]
|
|
2038
|
+
if (ir.kind === 'keywords') return ir.words
|
|
2039
|
+
return undefined
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
function lexicalIrEnd(ir: LexicalIr, input: string, start = 0): number | undefined {
|
|
2043
|
+
switch (ir.kind) {
|
|
2044
|
+
case 'literal': {
|
|
2045
|
+
const actual = input.slice(start, start + ir.value.length)
|
|
2046
|
+
return (ir.caseInsensitive ? foldAscii(actual) === foldAscii(ir.value) : actual === ir.value)
|
|
2047
|
+
? start + ir.value.length : undefined
|
|
2048
|
+
}
|
|
2049
|
+
case 'keywords': {
|
|
2050
|
+
for (const word of ir.words) {
|
|
2051
|
+
const actual = input.slice(start, start + word.length)
|
|
2052
|
+
if (ir.caseInsensitive ? foldAscii(actual) === foldAscii(word) : actual === word) {
|
|
2053
|
+
const next = input[start + word.length]
|
|
2054
|
+
if (next === undefined || ir.boundary === undefined) return start + word.length
|
|
2055
|
+
try {
|
|
2056
|
+
if (!new RegExp(`[${ir.boundary}]`).test(next)) return start + word.length
|
|
2057
|
+
} catch { return undefined }
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
2060
|
+
return undefined
|
|
2061
|
+
}
|
|
2062
|
+
case 'regex': {
|
|
2063
|
+
try {
|
|
2064
|
+
// regex() performs one authored sticky match. Anchoring the source in a
|
|
2065
|
+
// wrapper changes both alternation backtracking (`a|ab`) and multiline
|
|
2066
|
+
// `$` semantics, so proof must inspect the actual chosen end.
|
|
2067
|
+
const matcher = new RegExp(ir.source, `${regexFlags(ir.flags)}y`)
|
|
2068
|
+
matcher.lastIndex = start
|
|
2069
|
+
const match = matcher.exec(input)
|
|
2070
|
+
return match === null ? undefined : matcher.lastIndex
|
|
2071
|
+
} catch { return undefined }
|
|
2072
|
+
}
|
|
2073
|
+
case 'choice': {
|
|
2074
|
+
for (const arm of ir.arms) {
|
|
2075
|
+
const end = lexicalIrEnd(arm, input, start)
|
|
2076
|
+
if (end !== undefined) return end
|
|
2077
|
+
}
|
|
2078
|
+
return undefined
|
|
2079
|
+
}
|
|
2080
|
+
case 'sequence': {
|
|
2081
|
+
let end = start
|
|
2082
|
+
for (const part of ir.parts) {
|
|
2083
|
+
const next = lexicalIrEnd(part, input, end)
|
|
2084
|
+
if (next === undefined) return undefined
|
|
2085
|
+
end = next
|
|
2086
|
+
}
|
|
2087
|
+
return end
|
|
2088
|
+
}
|
|
2089
|
+
case 'repeat': {
|
|
2090
|
+
let end = start
|
|
2091
|
+
let count = 0
|
|
2092
|
+
while (ir.max === null || count < ir.max) {
|
|
2093
|
+
const next = lexicalIrEnd(ir.body, input, end)
|
|
2094
|
+
if (next === undefined || next === end) break
|
|
2095
|
+
end = next
|
|
2096
|
+
count++
|
|
2097
|
+
}
|
|
2098
|
+
return count >= ir.min ? end : undefined
|
|
2099
|
+
}
|
|
2100
|
+
case 'assert': {
|
|
2101
|
+
const matched = lexicalIrEnd(ir.body, input, start) !== undefined
|
|
2102
|
+
return matched === ir.positive ? start : undefined
|
|
2103
|
+
}
|
|
2104
|
+
case 'balanced': return undefined
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
|
|
2108
|
+
function lexicalIrAcceptsExact(ir: LexicalIr, value: string): boolean {
|
|
2109
|
+
return lexicalIrEnd(ir, value) === value.length
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
function asciiCaseVariants(value: string, caseInsensitive: boolean): readonly string[] | undefined {
|
|
2113
|
+
if (caseInsensitive && /[^\x00-\x7f]/.test(value)) return undefined
|
|
2114
|
+
if (!caseInsensitive || !/[A-Za-z]/.test(value)) return [value]
|
|
2115
|
+
let variants = ['']
|
|
2116
|
+
for (const char of value) {
|
|
2117
|
+
const lower = char >= 'A' && char <= 'Z' ? char.toLowerCase() : char
|
|
2118
|
+
const upper = char >= 'a' && char <= 'z' ? char.toUpperCase() : char
|
|
2119
|
+
if (lower !== upper && variants.length > 2048) return undefined
|
|
2120
|
+
variants = lower === upper
|
|
2121
|
+
? variants.map(prefix => prefix + char)
|
|
2122
|
+
: variants.flatMap(prefix => [prefix + lower, prefix + upper])
|
|
2123
|
+
}
|
|
2124
|
+
return variants
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
function regexAsciiCaseStable(source: string): boolean {
|
|
2128
|
+
let index = 0
|
|
2129
|
+
while (index < source.length) {
|
|
2130
|
+
const char = source[index]!
|
|
2131
|
+
if (char === '\\') {
|
|
2132
|
+
const kind = source[index + 1]
|
|
2133
|
+
if (kind === 'x') {
|
|
2134
|
+
const value = Number.parseInt(source.slice(index + 2, index + 4), 16)
|
|
2135
|
+
if (Number.isFinite(value) && /[A-Za-z]/.test(String.fromCharCode(value))) return false
|
|
2136
|
+
index += 4
|
|
2137
|
+
continue
|
|
2138
|
+
}
|
|
2139
|
+
if (kind === 'u') {
|
|
2140
|
+
const brace = source[index + 2] === '{'
|
|
2141
|
+
const end = brace ? source.indexOf('}', index + 3) : index + 6
|
|
2142
|
+
const digits = brace ? source.slice(index + 3, end) : source.slice(index + 2, end)
|
|
2143
|
+
const value = Number.parseInt(digits, 16)
|
|
2144
|
+
if (Number.isFinite(value) && value <= 0xffff
|
|
2145
|
+
&& /[A-Za-z]/.test(String.fromCharCode(value))) return false
|
|
2146
|
+
index = brace ? (end + 1 || source.length) : end
|
|
2147
|
+
continue
|
|
2148
|
+
}
|
|
2149
|
+
if (kind !== undefined && 'dDsSwWpP'.includes(kind)) { index += 2; continue }
|
|
2150
|
+
if (kind !== undefined && /[A-Za-z]/.test(kind)) return false
|
|
2151
|
+
index += 2
|
|
2152
|
+
continue
|
|
2153
|
+
}
|
|
2154
|
+
if (char === '[') {
|
|
2155
|
+
let end = index + 1
|
|
2156
|
+
for (; end < source.length; end++) {
|
|
2157
|
+
if (source[end] === '\\') { end++; continue }
|
|
2158
|
+
if (source[end] === ']') break
|
|
2159
|
+
}
|
|
2160
|
+
if (end >= source.length) return false
|
|
2161
|
+
const body = source.slice(index + 1, end)
|
|
2162
|
+
const lowerRange = body.includes('a-z')
|
|
2163
|
+
const upperRange = body.includes('A-Z')
|
|
2164
|
+
if (lowerRange !== upperRange) return false
|
|
2165
|
+
const withoutRanges = body.replace(/a-z|A-Z/g, '')
|
|
2166
|
+
const literals = new Set<string>()
|
|
2167
|
+
for (let at = 0; at < withoutRanges.length; at++) {
|
|
2168
|
+
if (withoutRanges[at] === '\\') {
|
|
2169
|
+
const kind = withoutRanges[++at]
|
|
2170
|
+
if (kind === 'x') {
|
|
2171
|
+
const value = Number.parseInt(withoutRanges.slice(at + 1, at + 3), 16)
|
|
2172
|
+
if (Number.isFinite(value) && /[A-Za-z]/.test(String.fromCharCode(value))) {
|
|
2173
|
+
literals.add(String.fromCharCode(value))
|
|
2174
|
+
}
|
|
2175
|
+
at += 2
|
|
2176
|
+
}
|
|
2177
|
+
else if (kind === 'u') {
|
|
2178
|
+
let digits: string
|
|
2179
|
+
if (withoutRanges[at + 1] === '{') {
|
|
2180
|
+
const end = withoutRanges.indexOf('}', at + 2)
|
|
2181
|
+
if (end < 0) return false
|
|
2182
|
+
digits = withoutRanges.slice(at + 2, end)
|
|
2183
|
+
at = end
|
|
2184
|
+
} else {
|
|
2185
|
+
digits = withoutRanges.slice(at + 1, at + 5)
|
|
2186
|
+
at += 4
|
|
2187
|
+
}
|
|
2188
|
+
const value = Number.parseInt(digits, 16)
|
|
2189
|
+
if (Number.isFinite(value) && value <= 0xffff
|
|
2190
|
+
&& /[A-Za-z]/.test(String.fromCharCode(value))) {
|
|
2191
|
+
literals.add(String.fromCharCode(value))
|
|
2192
|
+
}
|
|
2193
|
+
} else if (kind !== undefined && /[A-Za-z]/.test(kind)
|
|
2194
|
+
&& !'dDsSwWpP'.includes(kind)) return false
|
|
2195
|
+
continue
|
|
2196
|
+
}
|
|
2197
|
+
if (/[A-Za-z]/.test(withoutRanges[at]!)) literals.add(withoutRanges[at]!)
|
|
2198
|
+
}
|
|
2199
|
+
for (const letter of literals) {
|
|
2200
|
+
const peer = letter === letter.toLowerCase() ? letter.toUpperCase() : letter.toLowerCase()
|
|
2201
|
+
if (!literals.has(peer)) return false
|
|
2202
|
+
}
|
|
2203
|
+
index = end + 1
|
|
2204
|
+
continue
|
|
2205
|
+
}
|
|
2206
|
+
if (/[A-Za-z]/.test(char)) return false
|
|
2207
|
+
index++
|
|
2208
|
+
}
|
|
2209
|
+
return true
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
function lexicalIrAsciiCaseStable(ir: LexicalIr): boolean {
|
|
2213
|
+
switch (ir.kind) {
|
|
2214
|
+
case 'literal': return ir.caseInsensitive || !/[A-Za-z]/.test(ir.value)
|
|
2215
|
+
case 'keywords': return ir.caseInsensitive
|
|
2216
|
+
|| ir.words.every(word => !/[A-Za-z]/.test(word))
|
|
2217
|
+
case 'regex': return ir.flags.includes('i') || regexAsciiCaseStable(ir.source)
|
|
2218
|
+
case 'sequence': return ir.parts.every(lexicalIrAsciiCaseStable)
|
|
2219
|
+
case 'choice': return ir.arms.every(lexicalIrAsciiCaseStable)
|
|
2220
|
+
case 'repeat': case 'assert': return lexicalIrAsciiCaseStable(ir.body)
|
|
2221
|
+
case 'balanced': return !/[A-Za-z]/.test(ir.open + ir.close)
|
|
2222
|
+
&& ir.skip.every(lexicalIrAsciiCaseStable)
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
|
|
2226
|
+
type ContinuationClass = { readonly source: string; readonly flags: string }
|
|
2227
|
+
|
|
2228
|
+
function lexicalContinuationClasses(ir: LexicalIr): readonly ContinuationClass[] | undefined {
|
|
2229
|
+
if (ir.kind === 'regex') {
|
|
2230
|
+
const classesRemoved = ir.source.replace(/\[(?:\\.|[^\]\\])*\]/g, '[]')
|
|
2231
|
+
if (/[|()$^]/.test(classesRemoved) || /\\[bB]/.test(classesRemoved)) return undefined
|
|
2232
|
+
const tail = /(\[(?:\\.|[^\]\\])+\])(?:\*|\+)$/.exec(ir.source)
|
|
2233
|
+
if (tail === null || /[uv]/.test(ir.flags) || /\\[pP]|\\u\{/.test(tail[1]!)) return undefined
|
|
2234
|
+
return [{ source: tail[1]!, flags: ir.flags }]
|
|
2235
|
+
}
|
|
2236
|
+
if (ir.kind === 'sequence' && ir.parts.length === 2) {
|
|
2237
|
+
const [head, tail] = ir.parts
|
|
2238
|
+
if (tail?.kind !== 'repeat' || tail.min !== 0 || tail.max !== 1
|
|
2239
|
+
|| tail.body.kind !== 'literal' || [...tail.body.value].length !== 1) return undefined
|
|
2240
|
+
const headClasses = lexicalContinuationClasses(head!)
|
|
2241
|
+
if (headClasses === undefined) return undefined
|
|
2242
|
+
const escaped = tail.body.value.replace(/[\\\]\-^]/g, '\\$&')
|
|
2243
|
+
return [...headClasses, {
|
|
2244
|
+
source: `[${escaped}]`, flags: tail.body.caseInsensitive ? 'i' : '',
|
|
2245
|
+
}]
|
|
2246
|
+
}
|
|
2247
|
+
return undefined
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
function boundaryCoversContinuation(family: LexicalIr, boundary: string): boolean {
|
|
2251
|
+
const classes = lexicalContinuationClasses(family)
|
|
2252
|
+
if (classes === undefined) return false
|
|
2253
|
+
let boundaryMatcher: RegExp
|
|
2254
|
+
const matchers: RegExp[] = []
|
|
2255
|
+
try {
|
|
2256
|
+
boundaryMatcher = new RegExp(`^[${boundary}]$`)
|
|
2257
|
+
for (const entry of classes) {
|
|
2258
|
+
matchers.push(new RegExp(`^${entry.source}$`, regexFlags(entry.flags)))
|
|
2259
|
+
}
|
|
2260
|
+
} catch { return false }
|
|
2261
|
+
for (let code = 0; code <= 0xffff; code++) {
|
|
2262
|
+
const char = String.fromCharCode(code)
|
|
2263
|
+
if (matchers.some(matcher => matcher.test(char)) && !boundaryMatcher.test(char)) return false
|
|
2264
|
+
}
|
|
2265
|
+
return true
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
function fixedRegexLiteral(source: string): string | undefined {
|
|
2269
|
+
let value = ''
|
|
2270
|
+
for (let index = 0; index < source.length; index++) {
|
|
2271
|
+
const char = source[index]!
|
|
2272
|
+
if (char !== '\\') {
|
|
2273
|
+
if (/[.[\]{}()*+?|^$]/.test(char)) return undefined
|
|
2274
|
+
value += char
|
|
2275
|
+
continue
|
|
2276
|
+
}
|
|
2277
|
+
const kind = source[++index]
|
|
2278
|
+
if (kind === undefined || 'dDsSwWpPbB'.includes(kind)) return undefined
|
|
2279
|
+
if (kind === 'x') {
|
|
2280
|
+
const code = Number.parseInt(source.slice(index + 1, index + 3), 16)
|
|
2281
|
+
if (!Number.isFinite(code)) return undefined
|
|
2282
|
+
value += String.fromCharCode(code)
|
|
2283
|
+
index += 2
|
|
2284
|
+
continue
|
|
2285
|
+
}
|
|
2286
|
+
if (kind === 'u') {
|
|
2287
|
+
if (source[index + 1] === '{') return undefined
|
|
2288
|
+
const code = Number.parseInt(source.slice(index + 1, index + 5), 16)
|
|
2289
|
+
if (!Number.isFinite(code)) return undefined
|
|
2290
|
+
value += String.fromCharCode(code)
|
|
2291
|
+
index += 4
|
|
2292
|
+
continue
|
|
2293
|
+
}
|
|
2294
|
+
value += kind
|
|
2295
|
+
}
|
|
2296
|
+
return value
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
function familyHasContextFreeFiniteEnd(
|
|
2300
|
+
ir: LexicalIr,
|
|
2301
|
+
spellings: readonly string[],
|
|
2302
|
+
): boolean {
|
|
2303
|
+
if (ir.kind === 'literal') return true
|
|
2304
|
+
if (ir.kind === 'regex') return fixedRegexLiteral(ir.source) !== undefined
|
|
2305
|
+
if (ir.kind !== 'keywords' || ir.boundary !== undefined) return false
|
|
2306
|
+
return spellings.every(spelling => !ir.words.some(word => {
|
|
2307
|
+
const actual = ir.caseInsensitive ? foldAscii(spelling) : spelling
|
|
2308
|
+
const candidate = ir.caseInsensitive ? foldAscii(word) : word
|
|
2309
|
+
return candidate.length > actual.length && candidate.startsWith(actual)
|
|
2310
|
+
}))
|
|
2311
|
+
}
|
|
2312
|
+
|
|
2313
|
+
function compatibleLanguageViews(
|
|
2314
|
+
arm: LexicalIr,
|
|
2315
|
+
family: LexicalIr,
|
|
2316
|
+
): ReadonlyArray<{ readonly ir: LexicalIr; readonly relation: 'equal' | 'prefix' }> | undefined {
|
|
2317
|
+
if (JSON.stringify(arm) === JSON.stringify(family)) return [{ ir: arm, relation: 'equal' }]
|
|
2318
|
+
const values = finiteLexicalValues(arm)
|
|
2319
|
+
if (values === undefined || values.length === 0) return undefined
|
|
2320
|
+
const familyValues = finiteLexicalValues(family)
|
|
2321
|
+
const foldedArm = arm.kind === 'literal' ? arm.caseInsensitive : arm.kind === 'keywords' && arm.caseInsensitive
|
|
2322
|
+
const foldedFamily = family.kind === 'literal'
|
|
2323
|
+
? family.caseInsensitive : family.kind === 'keywords' && family.caseInsensitive
|
|
2324
|
+
const armBoundary = arm.kind === 'keywords' ? arm.boundary : undefined
|
|
2325
|
+
const views: Array<{ ir: LexicalIr; relation: 'equal' | 'prefix' }> = []
|
|
2326
|
+
for (const value of values) {
|
|
2327
|
+
let relation: 'equal' | 'prefix' | undefined
|
|
2328
|
+
const armCaseInsensitive = arm.kind === 'literal'
|
|
2329
|
+
? arm.caseInsensitive : arm.kind === 'keywords' && arm.caseInsensitive
|
|
2330
|
+
const spellings = armCaseInsensitive && !/[^\x00-\x7f]/.test(value)
|
|
2331
|
+
&& lexicalIrAsciiCaseStable(family)
|
|
2332
|
+
? [value] : asciiCaseVariants(value, armCaseInsensitive)
|
|
2333
|
+
const endIsTotal = spellings !== undefined
|
|
2334
|
+
&& (familyHasContextFreeFiniteEnd(family, spellings)
|
|
2335
|
+
|| armBoundary !== undefined && boundaryCoversContinuation(family, armBoundary))
|
|
2336
|
+
if (endIsTotal && spellings !== undefined
|
|
2337
|
+
&& spellings.every(spelling => lexicalIrAcceptsExact(family, spelling))) {
|
|
2338
|
+
relation = 'equal'
|
|
2339
|
+
}
|
|
2340
|
+
else if (familyValues !== undefined
|
|
2341
|
+
&& (family.kind === 'literal' || family.kind === 'keywords' && family.boundary === undefined)
|
|
2342
|
+
&& foldedArm === foldedFamily && familyValues.some(full => {
|
|
2343
|
+
const a = foldedArm ? foldAscii(value) : value
|
|
2344
|
+
const f = foldedFamily ? foldAscii(full) : full
|
|
2345
|
+
if (!f.startsWith(a) || f.length === a.length) return false
|
|
2346
|
+
if (arm.kind !== 'keywords' || arm.boundary === undefined) return true
|
|
2347
|
+
try { return !new RegExp(`[${arm.boundary}]`).test(full[a.length]!) } catch { return false }
|
|
2348
|
+
})) relation = 'prefix'
|
|
2349
|
+
if (relation === undefined) return undefined
|
|
2350
|
+
const ir: LexicalIr = arm.kind === 'keywords'
|
|
2351
|
+
? { ...arm, words: [value] }
|
|
2352
|
+
: arm.kind === 'literal'
|
|
2353
|
+
? { ...arm, value }
|
|
2354
|
+
: (() => { throw new Error('parseman: finite decision view lost its language') })()
|
|
2355
|
+
views.push({ ir, relation })
|
|
2356
|
+
}
|
|
2357
|
+
return views
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
type PendingDecisionView = {
|
|
2361
|
+
readonly familyKey: string
|
|
2362
|
+
readonly key: string
|
|
2363
|
+
readonly view: LexicalDecisionOutcomeView
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
type PendingAcceptance =
|
|
2367
|
+
| { readonly kind: 'outcomes'; readonly keys: readonly string[] }
|
|
2368
|
+
| { readonly kind: 'otherwise'; readonly excludingKeys: readonly string[] }
|
|
2369
|
+
| Exclude<LexicalDecisionAcceptance, { kind: 'outcomes' | 'otherwise' }>
|
|
2370
|
+
|
|
2371
|
+
type PendingDecisionSite = {
|
|
2372
|
+
readonly siteId: number
|
|
2373
|
+
readonly atom: 'choice' | 'dispatch'
|
|
2374
|
+
readonly path: string
|
|
2375
|
+
readonly contextKey: string
|
|
2376
|
+
readonly contextSnapshotId: number
|
|
2377
|
+
readonly families: ReadonlyArray<{
|
|
2378
|
+
readonly familyKey: string
|
|
2379
|
+
readonly arms: ReadonlyArray<Omit<LexicalDecisionArm, 'acceptance'> & { readonly acceptance: PendingAcceptance }>
|
|
2380
|
+
}>
|
|
2381
|
+
readonly precisionNotes: readonly string[]
|
|
2382
|
+
readonly pendingReuse: 'forbidden' | 'unproved'
|
|
2383
|
+
readonly reuseEpochs: readonly LexicalDecisionReuseEpoch[]
|
|
2384
|
+
readonly routeUsesRouted?: readonly boolean[]
|
|
2385
|
+
}
|
|
2386
|
+
|
|
2387
|
+
type ScanConsumerCandidate = {
|
|
2388
|
+
readonly path: string
|
|
2389
|
+
readonly contextKey: string
|
|
2390
|
+
readonly context: CapabilityContext
|
|
2391
|
+
readonly kind: 'scanTo' | 'balanced'
|
|
2392
|
+
readonly raw: boolean
|
|
2393
|
+
readonly ownSkip: readonly Combinator<unknown>[]
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
function balancedSpecOf(parser: Combinator<unknown>): BalancedSpec['_balancedSpec'] {
|
|
2397
|
+
let current = parser
|
|
2398
|
+
let spec = (current as BalancedSpec)._balancedSpec
|
|
2399
|
+
const seen = new Set<Combinator<unknown>>()
|
|
2400
|
+
while (spec === undefined && current._def.tag === 'token' && !seen.has(current)) {
|
|
2401
|
+
seen.add(current)
|
|
2402
|
+
current = current._def.parser
|
|
2403
|
+
spec = (current as BalancedSpec)._balancedSpec
|
|
2404
|
+
}
|
|
2405
|
+
return spec
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
function ownedScanConsumers(
|
|
2409
|
+
root: Combinator<unknown>,
|
|
2410
|
+
initialContext: CapabilityContext,
|
|
2411
|
+
contextKeyOf: (context: CapabilityContext) => string,
|
|
2412
|
+
isSeparateOccurrence: (parser: Combinator<unknown>, contextKey: string) => boolean,
|
|
2413
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
2414
|
+
): ScanConsumerCandidate[] {
|
|
2415
|
+
const out: ScanConsumerCandidate[] = []
|
|
2416
|
+
const stack = new Set<Combinator<unknown>>()
|
|
2417
|
+
const walk = (
|
|
2418
|
+
parser: Combinator<unknown>, path: string, first: boolean, context: CapabilityContext,
|
|
2419
|
+
): void => {
|
|
2420
|
+
if (stack.has(parser)) return
|
|
2421
|
+
const def = parser._def
|
|
2422
|
+
const contextKey = contextKeyOf(context)
|
|
2423
|
+
// Stop only at an independently inventoried occurrence. Nested atomic
|
|
2424
|
+
// wrappers inside token() are recognition machinery owned by the outer site.
|
|
2425
|
+
if (!first && isSeparateOccurrence(parser, contextKey)) return
|
|
2426
|
+
let childContext = context
|
|
2427
|
+
if (def.tag === 'grammar') childContext = {
|
|
2428
|
+
trivia: def.clearTrivia ? undefined : (def.triviaParser ?? context.trivia),
|
|
2429
|
+
scanSkip: context.scanSkip,
|
|
2430
|
+
trackLines: context.trackLines || def.trackLines,
|
|
2431
|
+
captureTrivia: context.captureTrivia || def.captureTrivia === true,
|
|
2432
|
+
opaqueRootCapture: context.opaqueRootCapture || def.rootCapture === 'opaque',
|
|
2433
|
+
dynamicState: context.dynamicState,
|
|
2434
|
+
}
|
|
2435
|
+
else if (def.tag === 'withCtx') childContext = { ...context, dynamicState: true }
|
|
2436
|
+
// token() clears trivia and collector buffers, but the public
|
|
2437
|
+
// ctx.captureTrivia property remains observable by authored callbacks.
|
|
2438
|
+
else if (def.tag === 'token') childContext = { ...context, trivia: undefined }
|
|
2439
|
+
else if (def.tag === 'trivia') childContext = { ...context, trivia: undefined }
|
|
2440
|
+
const balanced = balancedSpecOf(parser)
|
|
2441
|
+
if (balanced !== undefined) {
|
|
2442
|
+
out.push({
|
|
2443
|
+
path, contextKey: contextKeyOf(childContext), context: childContext,
|
|
2444
|
+
kind: 'balanced', raw: balanced.raw, ownSkip: balanced.ownSkip,
|
|
2445
|
+
})
|
|
2446
|
+
return
|
|
2447
|
+
}
|
|
2448
|
+
if (def.tag === 'scanTo') {
|
|
2449
|
+
// scanTo(raw) skips nothing, including its authored `skip` option.
|
|
2450
|
+
out.push({
|
|
2451
|
+
path, contextKey, context, kind: 'scanTo', raw: def.raw, ownSkip: def.raw ? [] : def.skip,
|
|
2452
|
+
})
|
|
2453
|
+
return
|
|
2454
|
+
}
|
|
2455
|
+
stack.add(parser)
|
|
2456
|
+
const children = tokenChildren(parser, resolve)
|
|
2457
|
+
for (let i = 0; i < children.length; i++) {
|
|
2458
|
+
walk(children[i]!, `${path}/${i}`, false, childContext)
|
|
2459
|
+
}
|
|
2460
|
+
stack.delete(parser)
|
|
2461
|
+
}
|
|
2462
|
+
walk(root, '', true, initialContext)
|
|
2463
|
+
return out
|
|
2464
|
+
}
|
|
2465
|
+
|
|
2466
|
+
function containsDynamicScanConsumer(
|
|
2467
|
+
root: Combinator<unknown>,
|
|
2468
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
2469
|
+
seen = new Set<Combinator<unknown>>(),
|
|
2470
|
+
): boolean {
|
|
2471
|
+
if (seen.has(root)) return false
|
|
2472
|
+
if (balancedSpecOf(root) !== undefined || root._def.tag === 'scanTo') return true
|
|
2473
|
+
seen.add(root)
|
|
2474
|
+
for (const child of tokenChildren(root, resolve)) {
|
|
2475
|
+
if (containsDynamicScanConsumer(child, resolve, seen)) return true
|
|
2476
|
+
}
|
|
2477
|
+
return false
|
|
2478
|
+
}
|
|
2479
|
+
|
|
2480
|
+
function predicateViews(def: Extract<ParserDef, { tag: 'dispatch' }>): Array<{
|
|
2481
|
+
readonly match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>
|
|
2482
|
+
readonly armId: number
|
|
2483
|
+
readonly parser: Combinator<unknown>
|
|
2484
|
+
readonly usesRouted: boolean
|
|
2485
|
+
}> {
|
|
2486
|
+
const out: Array<{
|
|
2487
|
+
match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>
|
|
2488
|
+
armId: number
|
|
2489
|
+
parser: Combinator<unknown>
|
|
2490
|
+
usesRouted: boolean
|
|
2491
|
+
}> = []
|
|
2492
|
+
let armId = 0
|
|
2493
|
+
for (const entry of def.cases) {
|
|
2494
|
+
for (const value of entry.keys) out.push({
|
|
2495
|
+
match: { kind: 'exact', values: [value], caseInsensitive: entry.caseInsensitive },
|
|
2496
|
+
armId, parser: entry.parser, usesRouted: branchUsesRouted(entry),
|
|
2497
|
+
})
|
|
2498
|
+
armId++
|
|
2499
|
+
}
|
|
2500
|
+
for (const entry of def.matchers ?? []) {
|
|
2501
|
+
out.push({
|
|
2502
|
+
match: entry.kind === 'matches'
|
|
2503
|
+
? { kind: 'matches', value: entry.value, flags: entry.flags ?? '', caseInsensitive: entry.caseInsensitive }
|
|
2504
|
+
: { kind: entry.kind, value: entry.value, caseInsensitive: entry.caseInsensitive },
|
|
2505
|
+
armId, parser: entry.parser, usesRouted: branchUsesRouted(entry),
|
|
2506
|
+
})
|
|
2507
|
+
armId++
|
|
2508
|
+
}
|
|
2509
|
+
return out
|
|
2510
|
+
}
|
|
2511
|
+
|
|
2512
|
+
function predicatePartitionStableFor(
|
|
2513
|
+
ir: LexicalIr,
|
|
2514
|
+
predicates: readonly Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[],
|
|
2515
|
+
): boolean {
|
|
2516
|
+
const folded = ir.kind === 'literal' ? ir.caseInsensitive : ir.kind === 'keywords' && ir.caseInsensitive
|
|
2517
|
+
const values = finiteLexicalValues(ir) ?? []
|
|
2518
|
+
if (!folded) return true
|
|
2519
|
+
if (values.some(value => /[^\x00-\x7f]/.test(value))) return false
|
|
2520
|
+
if (!values.some(value => /[A-Za-z]/.test(value))) return true
|
|
2521
|
+
return predicates.every(match => {
|
|
2522
|
+
if (match.kind === 'matches') return match.caseInsensitive || match.flags.includes('i')
|
|
2523
|
+
if (match.kind === 'exact') return match.caseInsensitive
|
|
2524
|
+
|| match.values.every(value => !/[A-Za-z]/.test(value))
|
|
2525
|
+
return match.caseInsensitive || !/[A-Za-z]/.test(match.value)
|
|
2526
|
+
})
|
|
2527
|
+
}
|
|
2528
|
+
|
|
2529
|
+
type CapabilityCandidate = {
|
|
2530
|
+
readonly bodyId: number
|
|
2531
|
+
readonly parser: Combinator<unknown>
|
|
2532
|
+
readonly atom: 'terminal' | 'token' | 'choice' | 'dispatch'
|
|
2533
|
+
readonly path: string
|
|
2534
|
+
readonly context: CapabilityContext
|
|
2535
|
+
readonly contextKey: string
|
|
2536
|
+
readonly recognitionContextKey: string
|
|
2537
|
+
}
|
|
2538
|
+
|
|
2539
|
+
type BindingEdgeCandidate = Omit<
|
|
2540
|
+
LexicalBindingEdge,
|
|
2541
|
+
'id' | 'status' | 'contextSnapshotId' | 'projectionId'
|
|
2542
|
+
> & {
|
|
2543
|
+
readonly parentBodyId: number
|
|
2544
|
+
readonly childBodyId: number
|
|
2545
|
+
readonly childOrdinal: number
|
|
2546
|
+
}
|
|
2547
|
+
|
|
2548
|
+
type CapabilityContext = LexicalCapabilityContext
|
|
2549
|
+
|
|
2550
|
+
/**
|
|
2551
|
+
* Enumerate atomic lexical ownership boundaries over the final graph. A token
|
|
2552
|
+
* owns its interior, so its private terminals are not separate obligations;
|
|
2553
|
+
* the same terminal reached independently outside that token remains one.
|
|
2554
|
+
*/
|
|
2555
|
+
function lexicalCapabilityCandidates(
|
|
2556
|
+
roots: ReadonlyArray<Combinator<unknown>>,
|
|
2557
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
2558
|
+
): {
|
|
2559
|
+
candidates: CapabilityCandidate[]
|
|
2560
|
+
bindingEdges: BindingEdgeCandidate[]
|
|
2561
|
+
contexts: Map<string, CapabilityContext>
|
|
2562
|
+
contextKeyOf: (context: CapabilityContext) => string
|
|
2563
|
+
} {
|
|
2564
|
+
const parserIds = new Map<Combinator<unknown>, number>()
|
|
2565
|
+
const parserId = (parser: Combinator<unknown>): number => {
|
|
2566
|
+
const prior = parserIds.get(parser)
|
|
2567
|
+
if (prior !== undefined) return prior
|
|
2568
|
+
const id = parserIds.size
|
|
2569
|
+
parserIds.set(parser, id)
|
|
2570
|
+
return id
|
|
2571
|
+
}
|
|
2572
|
+
const contextKeyOf = (context: CapabilityContext): string =>
|
|
2573
|
+
`t${context.trivia === undefined ? -1 : parserId(context.trivia)}`
|
|
2574
|
+
+ `/s${context.scanSkip.map(parserId).join(',')}`
|
|
2575
|
+
+ `/l${context.trackLines ? 1 : 0}`
|
|
2576
|
+
+ `/c${context.captureTrivia ? 1 : 0}`
|
|
2577
|
+
+ `/r${context.opaqueRootCapture ? 1 : 0}`
|
|
2578
|
+
+ `/d${context.dynamicState ? 1 : 0}`
|
|
2579
|
+
const topPath = new Map<Combinator<unknown>, Map<string, string>>()
|
|
2580
|
+
const ownedPath = new Map<Combinator<unknown>, Map<string, string>>()
|
|
2581
|
+
const candidates = new Map<string, CapabilityCandidate>()
|
|
2582
|
+
const bindingEdges = new Map<string, BindingEdgeCandidate>()
|
|
2583
|
+
const contexts = new Map<string, CapabilityContext>()
|
|
2584
|
+
const visit = (parser: Combinator<unknown>, path: string, owned: boolean, context: CapabilityContext): void => {
|
|
2585
|
+
const contextKey = contextKeyOf(context)
|
|
2586
|
+
if (!contexts.has(contextKey)) contexts.set(contextKey, context)
|
|
2587
|
+
const seen = owned ? ownedPath : topPath
|
|
2588
|
+
let paths = seen.get(parser)
|
|
2589
|
+
if (paths === undefined) { paths = new Map(); seen.set(parser, paths) }
|
|
2590
|
+
const prior = paths.get(contextKey)
|
|
2591
|
+
if (prior !== undefined && prior <= path) return
|
|
2592
|
+
paths.set(contextKey, path)
|
|
2593
|
+
|
|
2594
|
+
const def = parser._def
|
|
2595
|
+
const recognitionContext = def.tag === 'token'
|
|
2596
|
+
? { ...context, trivia: undefined, captureTrivia: false }
|
|
2597
|
+
: context
|
|
2598
|
+
const recognitionContextKey = contextKeyOf(recognitionContext)
|
|
2599
|
+
if (!owned && keyOf(def) !== undefined) {
|
|
2600
|
+
const key = `terminal\u0000${parserId(parser)}\u0000${contextKey}`
|
|
2601
|
+
const priorCandidate = candidates.get(key)
|
|
2602
|
+
if (priorCandidate === undefined || path < priorCandidate.path) {
|
|
2603
|
+
candidates.set(key, {
|
|
2604
|
+
bodyId: parserId(parser), parser, atom: 'terminal', path, context, contextKey, recognitionContextKey,
|
|
2605
|
+
})
|
|
2606
|
+
}
|
|
2607
|
+
} else if (!owned && (def.tag === 'token' || def.tag === 'choice' || def.tag === 'dispatch')) {
|
|
2608
|
+
const atom = def.tag
|
|
2609
|
+
const key = `${atom}\u0000${parserId(parser)}\u0000${contextKey}`
|
|
2610
|
+
const priorCandidate = candidates.get(key)
|
|
2611
|
+
if (priorCandidate === undefined || path < priorCandidate.path) {
|
|
2612
|
+
candidates.set(key, {
|
|
2613
|
+
bodyId: parserId(parser), parser, atom, path, context, contextKey, recognitionContextKey,
|
|
2614
|
+
})
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
// Exact atomic ownership: the token's child is recognition machinery, not
|
|
2618
|
+
// a separately linked parser site or fixed parent-edge obligation.
|
|
2619
|
+
if (!owned && def.tag === 'token') return
|
|
2620
|
+
const childOwned = owned || def.tag === 'token'
|
|
2621
|
+
const children = tokenChildren(parser, resolve)
|
|
2622
|
+
let childContext = context
|
|
2623
|
+
if (def.tag === 'grammar') {
|
|
2624
|
+
childContext = {
|
|
2625
|
+
trivia: def.clearTrivia ? undefined : (def.triviaParser ?? context.trivia),
|
|
2626
|
+
scanSkip: context.scanSkip,
|
|
2627
|
+
trackLines: context.trackLines || def.trackLines,
|
|
2628
|
+
captureTrivia: context.captureTrivia || def.captureTrivia === true,
|
|
2629
|
+
opaqueRootCapture: context.opaqueRootCapture || def.rootCapture === 'opaque',
|
|
2630
|
+
dynamicState: context.dynamicState,
|
|
2631
|
+
}
|
|
2632
|
+
} else if (def.tag === 'withCtx') {
|
|
2633
|
+
childContext = { ...context, dynamicState: true }
|
|
2634
|
+
} else if (def.tag === 'token') {
|
|
2635
|
+
childContext = { ...context, trivia: undefined }
|
|
2636
|
+
}
|
|
2637
|
+
for (let i = 0; i < children.length; i++) {
|
|
2638
|
+
const child = children[i]!
|
|
2639
|
+
const childPath = `${path}/${String(i).padStart(4, '0')}`
|
|
2640
|
+
const edgeKey = `${parserId(parser)}\u0000${i}\u0000${contextKey}`
|
|
2641
|
+
if (!bindingEdges.has(edgeKey)) bindingEdges.set(edgeKey, {
|
|
2642
|
+
path: childPath,
|
|
2643
|
+
contextKey, childContextKey: contextKeyOf(childContext),
|
|
2644
|
+
parentTag: def.tag,
|
|
2645
|
+
childTag: child._def.tag,
|
|
2646
|
+
parentBodyId: parserId(parser),
|
|
2647
|
+
childBodyId: parserId(child),
|
|
2648
|
+
childOrdinal: i,
|
|
2649
|
+
})
|
|
2650
|
+
visit(child, childPath, childOwned, childContext)
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2653
|
+
for (let i = 0; i < roots.length; i++) {
|
|
2654
|
+
const root = roots[i]!
|
|
2655
|
+
const rootPath = String(i).padStart(4, '0')
|
|
2656
|
+
const rootContext: CapabilityContext = {
|
|
2657
|
+
trivia: root._meta.grammarTrivia,
|
|
2658
|
+
scanSkip: root._meta.grammarScanSkip ?? [],
|
|
2659
|
+
trackLines: root._meta.grammarTrackLines === true,
|
|
2660
|
+
captureTrivia: false,
|
|
2661
|
+
opaqueRootCapture: false,
|
|
2662
|
+
dynamicState: false,
|
|
2663
|
+
}
|
|
2664
|
+
const contextKey = contextKeyOf(rootContext)
|
|
2665
|
+
bindingEdges.set(`root\u0000${i}\u0000${contextKey}`, {
|
|
2666
|
+
path: rootPath, contextKey, childContextKey: contextKey,
|
|
2667
|
+
parentTag: 'root', childTag: root._def.tag,
|
|
2668
|
+
parentBodyId: -1 - i, childBodyId: parserId(root), childOrdinal: i,
|
|
2669
|
+
})
|
|
2670
|
+
visit(root, rootPath, false, rootContext)
|
|
2671
|
+
}
|
|
2672
|
+
return {
|
|
2673
|
+
candidates: [...candidates.values()].sort((a, b) =>
|
|
2674
|
+
a.path < b.path ? -1 : a.path > b.path ? 1 : a.atom < b.atom ? -1 : a.atom > b.atom ? 1 : 0),
|
|
2675
|
+
bindingEdges: [...bindingEdges.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0),
|
|
2676
|
+
contexts,
|
|
2677
|
+
contextKeyOf,
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
|
|
2681
|
+
function decisionViewKey(view: LexicalDecisionOutcomeView): string {
|
|
2682
|
+
if (view.kind === 'whole') return 'whole'
|
|
2683
|
+
if (view.kind === 'predicate') return `predicate\u0000${canonicalLexicalOutcomeKey(view.match)}`
|
|
2684
|
+
return `language\u0000${view.relation}\u0000${JSON.stringify(view.ir)}`
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
function lexicalDecisionInventory(
|
|
2688
|
+
candidates: readonly CapabilityCandidate[],
|
|
2689
|
+
contextSnapshotIdByKey: ReadonlyMap<string, number>,
|
|
2690
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
2691
|
+
): {
|
|
2692
|
+
families: LexicalDecisionFamily[]
|
|
2693
|
+
outcomes: LexicalDecisionOutcome[]
|
|
2694
|
+
decisions: LexicalDecisionSite[]
|
|
2695
|
+
recognitionBySite: ReadonlyMap<number, LexicalCapabilityStatus>
|
|
2696
|
+
} {
|
|
2697
|
+
const familyIrByKey = new Map<string, LexicalIr>()
|
|
2698
|
+
const viewsByKey = new Map<string, PendingDecisionView>()
|
|
2699
|
+
const pending: PendingDecisionSite[] = []
|
|
2700
|
+
const addFamily = (lead: DecisionLead): string => {
|
|
2701
|
+
if (!familyIrByKey.has(lead.key)) familyIrByKey.set(lead.key, lead.ir)
|
|
2702
|
+
return lead.key
|
|
2703
|
+
}
|
|
2704
|
+
const addView = (familyKey: string, view: LexicalDecisionOutcomeView): string => {
|
|
2705
|
+
const key = `${familyKey}\u0000${decisionViewKey(view)}`
|
|
2706
|
+
if (!viewsByKey.has(key)) viewsByKey.set(key, { familyKey, key, view })
|
|
2707
|
+
return key
|
|
2708
|
+
}
|
|
2709
|
+
const reuseEpochsForChoice = (
|
|
2710
|
+
arms: readonly Combinator<unknown>[],
|
|
2711
|
+
): LexicalDecisionReuseEpoch[] => {
|
|
2712
|
+
const epochs: LexicalDecisionReuseEpoch[] = []
|
|
2713
|
+
let pure: number[] = []
|
|
2714
|
+
const flushPure = (): void => {
|
|
2715
|
+
if (pure.length === 0) return
|
|
2716
|
+
epochs.push({ armIds: pure, pendingReuse: 'unproved', boundary: 'none' })
|
|
2717
|
+
pure = []
|
|
2718
|
+
}
|
|
2719
|
+
for (let armId = 0; armId < arms.length; armId++) {
|
|
2720
|
+
if (!containsDynamicScanConsumer(arms[armId]!, resolve)) {
|
|
2721
|
+
pure.push(armId)
|
|
2722
|
+
continue
|
|
2723
|
+
}
|
|
2724
|
+
flushPure()
|
|
2725
|
+
epochs.push({ armIds: [armId], pendingReuse: 'forbidden', boundary: 'dynamic-scan' })
|
|
2726
|
+
}
|
|
2727
|
+
flushPure()
|
|
2728
|
+
return epochs
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2731
|
+
// The interner is grammar-wide, not derived from whichever decisions happen
|
|
2732
|
+
// to consume a language. This keeps family identity independent of cost/site
|
|
2733
|
+
// selection and gives the legacy token inventory an explicit relocation.
|
|
2734
|
+
for (const candidate of candidates) {
|
|
2735
|
+
const def = candidate.parser._def
|
|
2736
|
+
if (candidate.atom !== 'terminal' && candidate.atom !== 'token') continue
|
|
2737
|
+
const normalized = normalizeBalancedLexical(candidate.parser, resolve)
|
|
2738
|
+
?? normalizeLexical(def.tag === 'token' ? def.parser : candidate.parser, resolve, new Set())
|
|
2739
|
+
if (!('refusal' in normalized)) familyIrByKey.set(JSON.stringify(normalized.ir), normalized.ir)
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
for (let siteId = 0; siteId < candidates.length; siteId++) {
|
|
2743
|
+
const candidate = candidates[siteId]!
|
|
2744
|
+
if (candidate.atom !== 'choice' && candidate.atom !== 'dispatch') continue
|
|
2745
|
+
const def = candidate.parser._def
|
|
2746
|
+
const precisionNotes: string[] = []
|
|
2747
|
+
const contextSnapshotId = contextSnapshotIdByKey.get(candidate.contextKey)
|
|
2748
|
+
if (contextSnapshotId === undefined) throw new Error('parseman: decision occurrence lost its context snapshot')
|
|
2749
|
+
const dynamicScan = candidate.atom === 'dispatch' && def.tag === 'dispatch'
|
|
2750
|
+
? containsDynamicScanConsumer(def.selector, resolve)
|
|
2751
|
+
: containsDynamicScanConsumer(candidate.parser, resolve)
|
|
2752
|
+
if (dynamicScan) {
|
|
2753
|
+
precisionNotes.push('dynamic scan consumer forbids pending range reuse')
|
|
2754
|
+
}
|
|
2755
|
+
// A dispatch cannot classify before its selector when the selector itself is
|
|
2756
|
+
// a dynamic scan. Choice is different: a scan in one arm does not erase the
|
|
2757
|
+
// pure source-range languages of its siblings. Those arms remain
|
|
2758
|
+
// unrestricted and the SITE forbids pending reuse, but the compiler still
|
|
2759
|
+
// needs the sibling families to build and price a complete TOKEN body.
|
|
2760
|
+
if (candidate.atom === 'dispatch' && dynamicScan) {
|
|
2761
|
+
pending.push({
|
|
2762
|
+
siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
|
|
2763
|
+
contextSnapshotId, families: [], pendingReuse: 'forbidden',
|
|
2764
|
+
reuseEpochs: [{
|
|
2765
|
+
armIds: def.tag === 'dispatch'
|
|
2766
|
+
? Array.from({ length: def.cases.length + (def.matchers?.length ?? 0)
|
|
2767
|
+
+ (def.otherwise === undefined ? 0 : 1) }, (_, armId) => armId)
|
|
2768
|
+
: [],
|
|
2769
|
+
pendingReuse: 'forbidden', boundary: 'dynamic-scan',
|
|
2770
|
+
}],
|
|
2771
|
+
precisionNotes,
|
|
2772
|
+
})
|
|
2773
|
+
continue
|
|
2774
|
+
}
|
|
2775
|
+
if (candidate.atom === 'dispatch') {
|
|
2776
|
+
if (def.tag !== 'dispatch') throw new Error('parseman: decision occurrence lost its dispatch')
|
|
2777
|
+
const routeUsesRouted = [
|
|
2778
|
+
...def.cases.map(branchUsesRouted),
|
|
2779
|
+
...(def.matchers ?? []).map(branchUsesRouted),
|
|
2780
|
+
...(def.otherwise === undefined ? [] : [branchUsesRouted({
|
|
2781
|
+
parser: def.otherwise, usesRouted: def.otherwiseUsesRouted,
|
|
2782
|
+
})]),
|
|
2783
|
+
]
|
|
2784
|
+
const result = decisionLead(def.selector, candidate.context, resolve)
|
|
2785
|
+
if ('gap' in result) {
|
|
2786
|
+
precisionNotes.push(`selector: ${result.gap}`)
|
|
2787
|
+
pending.push({
|
|
2788
|
+
siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
|
|
2789
|
+
contextSnapshotId, families: [], precisionNotes, pendingReuse: 'unproved',
|
|
2790
|
+
reuseEpochs: [{
|
|
2791
|
+
armIds: Array.from({ length: def.cases.length + (def.matchers?.length ?? 0)
|
|
2792
|
+
+ (def.otherwise === undefined ? 0 : 1) }, (_, armId) => armId),
|
|
2793
|
+
pendingReuse: 'unproved', boundary: 'none',
|
|
2794
|
+
}],
|
|
2795
|
+
routeUsesRouted,
|
|
2796
|
+
})
|
|
2797
|
+
continue
|
|
2798
|
+
}
|
|
2799
|
+
const familyKey = addFamily(result.lead)
|
|
2800
|
+
const predicates = predicateViews(def)
|
|
2801
|
+
const predicateKeys = predicates.map(({ match }) => addView(familyKey, {
|
|
2802
|
+
kind: 'predicate', relation: 'equal', match,
|
|
2803
|
+
}))
|
|
2804
|
+
const arms: Array<Omit<LexicalDecisionArm, 'acceptance'> & { acceptance: PendingAcceptance }> = []
|
|
2805
|
+
let offset = 0
|
|
2806
|
+
for (let armId = 0; armId < def.cases.length; armId++) {
|
|
2807
|
+
const entry = def.cases[armId]!
|
|
2808
|
+
const keys = predicateKeys.slice(offset, offset + entry.keys.length)
|
|
2809
|
+
offset += entry.keys.length
|
|
2810
|
+
arms.push({
|
|
2811
|
+
armId, acceptance: { kind: 'outcomes', keys },
|
|
2812
|
+
usesRouted: branchUsesRouted(entry), dynamicGate: false,
|
|
2813
|
+
})
|
|
2814
|
+
}
|
|
2815
|
+
for (let i = 0; i < (def.matchers?.length ?? 0); i++) {
|
|
2816
|
+
const entry = def.matchers![i]!
|
|
2817
|
+
arms.push({
|
|
2818
|
+
armId: arms.length, acceptance: { kind: 'outcomes', keys: [predicateKeys[offset++]!] },
|
|
2819
|
+
usesRouted: branchUsesRouted(entry), dynamicGate: false,
|
|
2820
|
+
})
|
|
2821
|
+
}
|
|
2822
|
+
if (def.otherwise !== undefined) arms.push({
|
|
2823
|
+
armId: arms.length,
|
|
2824
|
+
acceptance: { kind: 'otherwise', excludingKeys: predicateKeys },
|
|
2825
|
+
usesRouted: branchUsesRouted({ parser: def.otherwise, usesRouted: def.otherwiseUsesRouted }),
|
|
2826
|
+
dynamicGate: false,
|
|
2827
|
+
})
|
|
2828
|
+
pending.push({
|
|
2829
|
+
siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
|
|
2830
|
+
contextSnapshotId, families: [{ familyKey, arms }], precisionNotes,
|
|
2831
|
+
pendingReuse: dynamicScan ? 'forbidden' : 'unproved',
|
|
2832
|
+
reuseEpochs: [{
|
|
2833
|
+
armIds: arms.map(arm => arm.armId),
|
|
2834
|
+
pendingReuse: dynamicScan ? 'forbidden' : 'unproved',
|
|
2835
|
+
boundary: dynamicScan ? 'dynamic-scan' : 'none',
|
|
2836
|
+
}],
|
|
2837
|
+
routeUsesRouted,
|
|
2838
|
+
})
|
|
2839
|
+
continue
|
|
2840
|
+
}
|
|
2841
|
+
|
|
2842
|
+
if (def.tag !== 'choice') throw new Error('parseman: decision occurrence lost its choice')
|
|
2843
|
+
const leads = def.parsers.map((arm, armId) => {
|
|
2844
|
+
const result = decisionLead(arm, candidate.context, resolve)
|
|
2845
|
+
if ('gap' in result) precisionNotes.push(`arm ${armId}: ${result.gap}`)
|
|
2846
|
+
return result
|
|
2847
|
+
})
|
|
2848
|
+
const familyKeys = [...new Set(leads.flatMap(result => 'lead' in result
|
|
2849
|
+
? [addFamily(result.lead)] : []))].sort()
|
|
2850
|
+
if (familyKeys.length === 0 && precisionNotes.length === 0) {
|
|
2851
|
+
precisionNotes.push('choice has no predecision range family; all arms remain unrestricted')
|
|
2852
|
+
}
|
|
2853
|
+
const families = familyKeys.map(familyKey => {
|
|
2854
|
+
const familyIr = familyIrByKey.get(familyKey)!
|
|
2855
|
+
const wholeKey = addView(familyKey, { kind: 'whole', relation: 'equal' })
|
|
2856
|
+
const familyDispatches = [...new Set(leads.flatMap(result =>
|
|
2857
|
+
'lead' in result && result.lead.key === familyKey && result.lead.dispatch !== undefined
|
|
2858
|
+
? [result.lead.dispatch] : []))]
|
|
2859
|
+
const ambiguousPartition = familyDispatches.length > 1
|
|
2860
|
+
if (ambiguousPartition) precisionNotes.push(
|
|
2861
|
+
`family ${familyKey}: multiple dispatches need distinct site-local outcome partitions`)
|
|
2862
|
+
const familyPredicates = ambiguousPartition ? []
|
|
2863
|
+
: familyDispatches.flatMap(dispatch => predicateViews(dispatch).map(entry => entry.match))
|
|
2864
|
+
const predicateKeys = familyPredicates.map(match => addView(familyKey, {
|
|
2865
|
+
kind: 'predicate', relation: 'equal', match,
|
|
2866
|
+
}))
|
|
2867
|
+
const arms = def.parsers.map((arm, armId): Omit<LexicalDecisionArm, 'acceptance'> & {
|
|
2868
|
+
acceptance: PendingAcceptance
|
|
2869
|
+
} => {
|
|
2870
|
+
const dynamicGate = def.gates[armId] !== null || def.autoNot[armId] !== null
|
|
2871
|
+
const result = leads[armId]!
|
|
2872
|
+
if ('gap' in result) return {
|
|
2873
|
+
armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate,
|
|
2874
|
+
}
|
|
2875
|
+
const lead = result.lead
|
|
2876
|
+
if (lead.key === familyKey) {
|
|
2877
|
+
if (ambiguousPartition) return {
|
|
2878
|
+
armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate,
|
|
2879
|
+
}
|
|
2880
|
+
if (lead.dispatch !== undefined) {
|
|
2881
|
+
const predicates = predicateViews(lead.dispatch)
|
|
2882
|
+
const keys = predicates.map(({ match }) => addView(familyKey, {
|
|
2883
|
+
kind: 'predicate', relation: 'equal', match,
|
|
2884
|
+
}))
|
|
2885
|
+
return {
|
|
2886
|
+
armId,
|
|
2887
|
+
acceptance: lead.dispatch.otherwise === undefined
|
|
2888
|
+
? { kind: 'outcomes', keys }
|
|
2889
|
+
: { kind: 'unrestricted' },
|
|
2890
|
+
usesRouted: false,
|
|
2891
|
+
dynamicGate,
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
return {
|
|
2895
|
+
armId, acceptance: { kind: 'outcomes', keys: [wholeKey] },
|
|
2896
|
+
usesRouted: false, dynamicGate,
|
|
2897
|
+
}
|
|
2898
|
+
}
|
|
2899
|
+
const familyLead = leads.find((entry): entry is { lead: DecisionLead } =>
|
|
2900
|
+
'lead' in entry && entry.lead.key === familyKey)!.lead
|
|
2901
|
+
const armFirst = firstSetOf(arm, new Set(), resolve)
|
|
2902
|
+
const familyFirst = firstSetOf(familyLead.parser, new Set(), resolve)
|
|
2903
|
+
if (armFirst.kind === 'ranges' && familyFirst.kind === 'ranges'
|
|
2904
|
+
&& !intersects(armFirst, familyFirst)) {
|
|
2905
|
+
return { armId, acceptance: { kind: 'impossible' }, usesRouted: false, dynamicGate }
|
|
2906
|
+
}
|
|
2907
|
+
const compatible = compatibleLanguageViews(lead.ir, familyIr)
|
|
2908
|
+
if (compatible !== undefined) {
|
|
2909
|
+
if (compatible.every(view => view.relation === 'equal')
|
|
2910
|
+
&& familyPredicates.length > 0
|
|
2911
|
+
&& predicatePartitionStableFor(lead.ir, familyPredicates)) {
|
|
2912
|
+
const values = finiteLexicalValues(lead.ir)!
|
|
2913
|
+
const accepted = new Set<string>()
|
|
2914
|
+
let hasOtherwise = false
|
|
2915
|
+
for (const value of values) {
|
|
2916
|
+
const matching = familyPredicates.flatMap((match, index) =>
|
|
2917
|
+
lexicalOutcomeMatches(match, value, 0, value.length) ? [predicateKeys[index]!] : [])
|
|
2918
|
+
if (matching.length === 0) hasOtherwise = true
|
|
2919
|
+
else for (const key of matching) accepted.add(key)
|
|
2920
|
+
}
|
|
2921
|
+
if (hasOtherwise && accepted.size === 0) return {
|
|
2922
|
+
armId,
|
|
2923
|
+
acceptance: { kind: 'otherwise', excludingKeys: predicateKeys },
|
|
2924
|
+
usesRouted: false,
|
|
2925
|
+
dynamicGate,
|
|
2926
|
+
}
|
|
2927
|
+
if (!hasOtherwise && accepted.size > 0) return {
|
|
2928
|
+
armId,
|
|
2929
|
+
acceptance: { kind: 'outcomes', keys: [...accepted] },
|
|
2930
|
+
usesRouted: false,
|
|
2931
|
+
dynamicGate,
|
|
2932
|
+
}
|
|
2933
|
+
// A finite arm split across direct predicates and the local
|
|
2934
|
+
// complement is conservatively retained; neither half is dropped.
|
|
2935
|
+
return { armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate }
|
|
2936
|
+
}
|
|
2937
|
+
const keys = compatible.map(view => addView(familyKey, {
|
|
2938
|
+
kind: 'language', relation: view.relation, ir: view.ir,
|
|
2939
|
+
}))
|
|
2940
|
+
return {
|
|
2941
|
+
armId, acceptance: { kind: 'outcomes', keys },
|
|
2942
|
+
usesRouted: false, dynamicGate,
|
|
2943
|
+
}
|
|
2944
|
+
}
|
|
2945
|
+
return { armId, acceptance: { kind: 'unrestricted' }, usesRouted: false, dynamicGate }
|
|
2946
|
+
})
|
|
2947
|
+
return { familyKey, arms }
|
|
2948
|
+
})
|
|
2949
|
+
pending.push({
|
|
2950
|
+
siteId, atom: candidate.atom, path: candidate.path, contextKey: candidate.contextKey,
|
|
2951
|
+
contextSnapshotId, families, precisionNotes: [...new Set(precisionNotes)].sort(),
|
|
2952
|
+
pendingReuse: dynamicScan ? 'forbidden' : 'unproved',
|
|
2953
|
+
reuseEpochs: reuseEpochsForChoice(def.parsers),
|
|
2954
|
+
})
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2957
|
+
const familyKeys = [...familyIrByKey.keys()].sort()
|
|
2958
|
+
const familyIdByKey = new Map(familyKeys.map((key, index) => [key, FIRST_LEXICAL_FAMILY_ID + index]))
|
|
2959
|
+
const families = familyKeys.map((semanticKey, index): LexicalDecisionFamily => ({
|
|
2960
|
+
id: FIRST_LEXICAL_FAMILY_ID + index, semanticKey, ir: familyIrByKey.get(semanticKey)!,
|
|
2961
|
+
}))
|
|
2962
|
+
const viewKeys = [...viewsByKey.keys()].sort()
|
|
2963
|
+
const firstOutcomeId = FIRST_LEXICAL_FAMILY_ID + familyKeys.length
|
|
2964
|
+
const outcomeIdByKey = new Map(viewKeys.map((key, index) => [key, firstOutcomeId + index]))
|
|
2965
|
+
const outcomes = viewKeys.map((key, index): LexicalDecisionOutcome => {
|
|
2966
|
+
const pendingView = viewsByKey.get(key)!
|
|
2967
|
+
return {
|
|
2968
|
+
id: firstOutcomeId + index,
|
|
2969
|
+
familyId: familyIdByKey.get(pendingView.familyKey)!,
|
|
2970
|
+
semanticKey: decisionViewKey(pendingView.view),
|
|
2971
|
+
view: pendingView.view,
|
|
2972
|
+
}
|
|
2973
|
+
})
|
|
2974
|
+
const decisions = pending.map((site): LexicalDecisionSite => ({
|
|
2975
|
+
siteId: site.siteId, atom: site.atom, path: site.path, contextKey: site.contextKey,
|
|
2976
|
+
contextSnapshotId: site.contextSnapshotId,
|
|
2977
|
+
fallback: 'unrestricted', precisionNotes: site.precisionNotes, pendingReuse: site.pendingReuse,
|
|
2978
|
+
reuseEpochs: site.reuseEpochs,
|
|
2979
|
+
...(site.routeUsesRouted === undefined ? {} : { routeUsesRouted: site.routeUsesRouted }),
|
|
2980
|
+
families: site.families.map(family => ({
|
|
2981
|
+
familyId: familyIdByKey.get(family.familyKey)!,
|
|
2982
|
+
arms: family.arms.map(arm => ({
|
|
2983
|
+
...arm,
|
|
2984
|
+
acceptance: arm.acceptance.kind === 'outcomes'
|
|
2985
|
+
? { kind: 'outcomes', outcomeIds: [...new Set(arm.acceptance.keys.map(key => outcomeIdByKey.get(key)!))].sort((a, b) => a - b) }
|
|
2986
|
+
: arm.acceptance.kind === 'otherwise'
|
|
2987
|
+
? { kind: 'otherwise', excludingOutcomeIds: [...new Set(arm.acceptance.excludingKeys.map(key => outcomeIdByKey.get(key)!))].sort((a, b) => a - b) }
|
|
2988
|
+
: arm.acceptance,
|
|
2989
|
+
})),
|
|
2990
|
+
})),
|
|
2991
|
+
}))
|
|
2992
|
+
const recognitionBySite = new Map<number, LexicalCapabilityStatus>(decisions.map(site => [
|
|
2993
|
+
site.siteId, COMPLETE_CAPABILITY,
|
|
2994
|
+
]))
|
|
2995
|
+
return { families, outcomes, decisions, recognitionBySite }
|
|
2996
|
+
}
|
|
2997
|
+
|
|
2998
|
+
function classifiedLiteralProjection(parser: Combinator<unknown>): {
|
|
2999
|
+
spelling: string
|
|
3000
|
+
transforms: readonly {
|
|
3001
|
+
fn: (value: unknown, span: { start: number; end: number }) => unknown
|
|
3002
|
+
source?: string
|
|
3003
|
+
}[]
|
|
3004
|
+
} | undefined {
|
|
3005
|
+
let current = parser
|
|
3006
|
+
const outerFirst: Array<{
|
|
3007
|
+
fn: (value: unknown, span: { start: number; end: number }) => unknown
|
|
3008
|
+
source?: string
|
|
3009
|
+
}> = []
|
|
3010
|
+
const seen = new Set<Combinator<unknown>>()
|
|
3011
|
+
while (!seen.has(current)) {
|
|
3012
|
+
seen.add(current)
|
|
3013
|
+
const def = current._def
|
|
3014
|
+
if (def.tag === 'literal') return { spelling: def.value, transforms: outerFirst.reverse() }
|
|
3015
|
+
if (def.tag === 'transform') {
|
|
3016
|
+
outerFirst.push({ fn: def.fn, ...(def.fnSrc === undefined ? {} : { source: def.fnSrc }) })
|
|
3017
|
+
current = def.parser
|
|
3018
|
+
continue
|
|
3019
|
+
}
|
|
3020
|
+
if (def.tag === 'label') { current = def.parser; continue }
|
|
3021
|
+
return undefined
|
|
3022
|
+
}
|
|
3023
|
+
return undefined
|
|
3024
|
+
}
|
|
3025
|
+
|
|
3026
|
+
function decisionEffectInventory(
|
|
3027
|
+
candidates: readonly CapabilityCandidate[],
|
|
3028
|
+
decisions: readonly LexicalDecisionSite[],
|
|
3029
|
+
bindingEdges: readonly LexicalBindingEdge[],
|
|
3030
|
+
bindingProjections: readonly LexicalBindingProjection[],
|
|
3031
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
3032
|
+
): LexicalDecisionEffectInventory {
|
|
3033
|
+
const finalDecisionAuthorities: LexicalFinalDecisionAuthority[] = []
|
|
3034
|
+
const decisionEffects: LexicalDecisionEffectProgram[] = []
|
|
3035
|
+
const decisionChildEffects: LexicalDecisionChildEffect[] = []
|
|
3036
|
+
const choiceClassAuthorities: LexicalChoiceClassAuthority[] = []
|
|
3037
|
+
const choiceExecutionOrders: LexicalChoiceExecutionOrder[] = []
|
|
3038
|
+
const classifySpellingMaps: LexicalClassifySpellingMap[] = []
|
|
3039
|
+
const classifyProjectionPlans: LexicalClassifyProjectionPlan[] = []
|
|
3040
|
+
const decisionExpectedAuthorities: LexicalExpectedAuthority[] = []
|
|
3041
|
+
const decisionCallbackAuthorities: LexicalDecisionCallbackAuthority[] = []
|
|
3042
|
+
const expectedIdByKey = new Map<string, number>()
|
|
3043
|
+
const callbackIds = new Map<LexicalDecisionCallbackAuthority['callback'], number>()
|
|
3044
|
+
const sourceIds = new Map<string, number>()
|
|
3045
|
+
const autoNotIds = new Map<string, number>()
|
|
3046
|
+
const edgeByPath = new Map(bindingEdges.map(edge => [edge.path, edge.id]))
|
|
3047
|
+
const expectedId = (values: readonly string[]): number => {
|
|
3048
|
+
const key = JSON.stringify(values)
|
|
3049
|
+
const prior = expectedIdByKey.get(key)
|
|
3050
|
+
if (prior !== undefined) return prior
|
|
3051
|
+
const id = decisionExpectedAuthorities.length
|
|
3052
|
+
expectedIdByKey.set(key, id)
|
|
3053
|
+
decisionExpectedAuthorities.push({ id, values: [...values] })
|
|
3054
|
+
return id
|
|
3055
|
+
}
|
|
3056
|
+
const callbackId = (value: LexicalDecisionCallbackAuthority['callback']): number => {
|
|
3057
|
+
const prior = callbackIds.get(value)
|
|
3058
|
+
if (prior !== undefined) return prior
|
|
3059
|
+
const id = decisionCallbackAuthorities.length
|
|
3060
|
+
callbackIds.set(value, id)
|
|
3061
|
+
decisionCallbackAuthorities.push({ id, callback: value })
|
|
3062
|
+
return id
|
|
3063
|
+
}
|
|
3064
|
+
const stringId = (map: Map<string, number>, value: string): number => {
|
|
3065
|
+
const prior = map.get(value)
|
|
3066
|
+
if (prior !== undefined) return prior
|
|
3067
|
+
const id = map.size
|
|
3068
|
+
map.set(value, id)
|
|
3069
|
+
return id
|
|
3070
|
+
}
|
|
3071
|
+
const edgeAt = (path: string, childIndex: number): number => {
|
|
3072
|
+
const childPath = `${path}/${String(childIndex).padStart(4, '0')}`
|
|
3073
|
+
const edge = edgeByPath.get(childPath)
|
|
3074
|
+
if (edge === undefined) throw new Error(`parseman: decision child lost binding edge ${childPath}`)
|
|
3075
|
+
return edge
|
|
3076
|
+
}
|
|
3077
|
+
const childEffect = (
|
|
3078
|
+
siteId: number,
|
|
3079
|
+
armId: number,
|
|
3080
|
+
childBindingEdgeId: number,
|
|
3081
|
+
parser: Combinator<unknown>,
|
|
3082
|
+
gate: ((state: unknown) => boolean) | null = null,
|
|
3083
|
+
gateSource: string | null = null,
|
|
3084
|
+
autoNot: readonly AutoNotCheck[] | null = null,
|
|
3085
|
+
): number => {
|
|
3086
|
+
const id = decisionChildEffects.length
|
|
3087
|
+
decisionChildEffects.push({
|
|
3088
|
+
id, decisionSiteId: siteId, armId, childBindingEdgeId,
|
|
3089
|
+
expectedAuthorityId: expectedId(deriveExpected(parser)),
|
|
3090
|
+
...(gate === null ? {} : { gateCallbackId: callbackId(gate) }),
|
|
3091
|
+
...(gateSource === null ? {} : { gateSourceId: stringId(sourceIds, gateSource) }),
|
|
3092
|
+
...(autoNot === null ? {} : { autoNotAuthorityId: stringId(autoNotIds, JSON.stringify(autoNot)) }),
|
|
3093
|
+
})
|
|
3094
|
+
return id
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
for (const decision of decisions) {
|
|
3098
|
+
const candidate = candidates[decision.siteId]
|
|
3099
|
+
if (candidate === undefined || candidate.parser._def.tag !== decision.atom) {
|
|
3100
|
+
throw new Error(`parseman: decision effect lost occurrence ${decision.siteId}`)
|
|
3101
|
+
}
|
|
3102
|
+
const def = candidate.parser._def
|
|
3103
|
+
const authorityId = finalDecisionAuthorities.length
|
|
3104
|
+
const effectId = decisionEffects.length
|
|
3105
|
+
let loweringGap: string | undefined
|
|
3106
|
+
const childEffectIds: number[] = []
|
|
3107
|
+
if (def.tag === 'choice') {
|
|
3108
|
+
const finalChoice = classifyFinalChoice(def.parsers, resolve)
|
|
3109
|
+
const authoredArmIds = def.parsers.map((_, armId) => armId)
|
|
3110
|
+
let mode: Extract<LexicalFinalDecisionAuthority, { atom: 'choice' }>['mode']
|
|
3111
|
+
if (finalChoice.exclusive) {
|
|
3112
|
+
const classAuthorityId = choiceClassAuthorities.length
|
|
3113
|
+
choiceClassAuthorities.push({
|
|
3114
|
+
id: classAuthorityId, armIds: authoredArmIds, firstSets: finalChoice.firstSets,
|
|
3115
|
+
})
|
|
3116
|
+
mode = { kind: 'exclusive', classAuthorityId }
|
|
3117
|
+
} else if (def.strategy.tag === 'literalsLongestFirst') {
|
|
3118
|
+
const orderAuthorityId = choiceExecutionOrders.length
|
|
3119
|
+
choiceExecutionOrders.push({
|
|
3120
|
+
id: orderAuthorityId, armIds: [...def.strategy.sortedIndices],
|
|
3121
|
+
})
|
|
3122
|
+
mode = { kind: 'longest', orderAuthorityId }
|
|
3123
|
+
} else if (def.strategy.tag === 'greedyClassify') {
|
|
3124
|
+
const spellingMapAuthorityId = classifySpellingMaps.length
|
|
3125
|
+
const projectionAuthorityId = classifyProjectionPlans.length
|
|
3126
|
+
const spellingEntries: Array<{ spelling: string; armId: number }> = []
|
|
3127
|
+
const projectionEntries: Array<{
|
|
3128
|
+
armId: number
|
|
3129
|
+
transformIds: readonly number[]
|
|
3130
|
+
transformSourceIds: readonly (number | -1)[]
|
|
3131
|
+
transformCount: number
|
|
3132
|
+
}> = []
|
|
3133
|
+
for (let armId = 0; armId < def.parsers.length; armId++) {
|
|
3134
|
+
if (armId === def.strategy.superIndex) continue
|
|
3135
|
+
const projection = classifiedLiteralProjection(def.parsers[armId]!)
|
|
3136
|
+
if (projection === undefined) {
|
|
3137
|
+
throw new Error(`parseman: greedy classifier lost literal arm ${armId}`)
|
|
3138
|
+
}
|
|
3139
|
+
spellingEntries.push({ spelling: projection.spelling, armId })
|
|
3140
|
+
const transformIds = projection.transforms.map(entry => callbackId(entry.fn))
|
|
3141
|
+
const transformSourceIds = projection.transforms.map(entry =>
|
|
3142
|
+
entry.source === undefined ? -1 : stringId(sourceIds, entry.source))
|
|
3143
|
+
projectionEntries.push({
|
|
3144
|
+
armId, transformIds, transformSourceIds, transformCount: transformIds.length,
|
|
3145
|
+
})
|
|
3146
|
+
if (transformSourceIds.includes(-1)) {
|
|
3147
|
+
loweringGap = 'classified transform has no statically named reader projection'
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
classifySpellingMaps.push({ id: spellingMapAuthorityId, entries: spellingEntries })
|
|
3151
|
+
classifyProjectionPlans.push({ id: projectionAuthorityId, entries: projectionEntries })
|
|
3152
|
+
mode = {
|
|
3153
|
+
kind: 'classify', superArmId: def.strategy.superIndex,
|
|
3154
|
+
spellingMapAuthorityId, projectionAuthorityId,
|
|
3155
|
+
}
|
|
3156
|
+
} else mode = { kind: 'ordered' }
|
|
3157
|
+
const expectedParsers = mode.kind === 'longest'
|
|
3158
|
+
? mode.orderAuthorityId === undefined ? def.parsers
|
|
3159
|
+
: choiceExecutionOrders[mode.orderAuthorityId]!.armIds.map(armId => def.parsers[armId]!)
|
|
3160
|
+
: mode.kind === 'classify' ? [def.parsers[mode.superArmId]!]
|
|
3161
|
+
: mode.kind === 'exclusive' ? def.parsers : undefined
|
|
3162
|
+
finalDecisionAuthorities.push({
|
|
3163
|
+
id: authorityId, atom: 'choice', decisionSiteId: decision.siteId,
|
|
3164
|
+
authoredArmIds, mode,
|
|
3165
|
+
...(expectedParsers === undefined ? {} : {
|
|
3166
|
+
staticMissExpectedAuthorityId: expectedId(
|
|
3167
|
+
expectedParsers.flatMap(parser => deriveExpected(parser)),
|
|
3168
|
+
),
|
|
3169
|
+
}),
|
|
3170
|
+
})
|
|
3171
|
+
for (let armId = 0; armId < def.parsers.length; armId++) {
|
|
3172
|
+
const gate = def.gates[armId] ?? null
|
|
3173
|
+
const gateSource = def.gateSrcs?.[armId] ?? null
|
|
3174
|
+
if (gate !== null && gateSource === null) {
|
|
3175
|
+
loweringGap = 'dynamic choice gate has no statically named reader projection'
|
|
3176
|
+
}
|
|
3177
|
+
childEffectIds.push(childEffect(
|
|
3178
|
+
decision.siteId, armId, edgeAt(candidate.path, armId), def.parsers[armId]!,
|
|
3179
|
+
gate, gateSource, def.autoNot[armId],
|
|
3180
|
+
))
|
|
3181
|
+
}
|
|
3182
|
+
} else {
|
|
3183
|
+
const routeParsers = [
|
|
3184
|
+
...def.cases.map(entry => entry.parser),
|
|
3185
|
+
...(def.matchers ?? []).map(entry => entry.parser),
|
|
3186
|
+
...(def.otherwise === undefined ? [] : [def.otherwise]),
|
|
3187
|
+
]
|
|
3188
|
+
const routeBranches = [
|
|
3189
|
+
...def.cases,
|
|
3190
|
+
...(def.matchers ?? []),
|
|
3191
|
+
...(def.otherwise === undefined ? [] : [{
|
|
3192
|
+
parser: def.otherwise, usesRouted: def.otherwiseUsesRouted,
|
|
3193
|
+
}]),
|
|
3194
|
+
]
|
|
3195
|
+
const otherwiseOffset = def.otherwise === undefined ? 0 : 1
|
|
3196
|
+
const routeEdgeIndices = [
|
|
3197
|
+
...def.cases.map((_, index) => 1 + otherwiseOffset + index),
|
|
3198
|
+
...(def.matchers ?? []).map((_, index) => 1 + otherwiseOffset + def.cases.length + index),
|
|
3199
|
+
...(def.otherwise === undefined ? [] : [1]),
|
|
3200
|
+
]
|
|
3201
|
+
const selectorBindingEdgeId = edgeAt(candidate.path, 0)
|
|
3202
|
+
const orderedRouteIds = routeParsers.map((_, routeId) => routeId)
|
|
3203
|
+
finalDecisionAuthorities.push({
|
|
3204
|
+
id: authorityId, atom: 'dispatch', decisionSiteId: decision.siteId,
|
|
3205
|
+
selectorBindingEdgeId,
|
|
3206
|
+
// The Stage-A occurrence owns exact/fold/matcher/otherwise precedence.
|
|
3207
|
+
classifierAuthorityId: decision.siteId,
|
|
3208
|
+
orderedRouteIds,
|
|
3209
|
+
noRouteExpectedAuthorityId: expectedId(def.cases.flatMap(entry =>
|
|
3210
|
+
entry.keys.map(key => JSON.stringify(key)))),
|
|
3211
|
+
})
|
|
3212
|
+
for (let routeId = 0; routeId < routeParsers.length; routeId++) {
|
|
3213
|
+
childEffectIds.push(childEffect(
|
|
3214
|
+
decision.siteId, routeId, edgeAt(candidate.path, routeEdgeIndices[routeId]!),
|
|
3215
|
+
routeParsers[routeId]!,
|
|
3216
|
+
))
|
|
3217
|
+
}
|
|
3218
|
+
const routedAuthority = decision.routeUsesRouted
|
|
3219
|
+
if (routedAuthority === undefined || routedAuthority.length !== routeBranches.length
|
|
3220
|
+
|| routedAuthority.some((value, index) => value !== branchUsesRouted(routeBranches[index]!))) {
|
|
3221
|
+
loweringGap = 'dispatch routed policy lost its Stage-A authority'
|
|
3222
|
+
}
|
|
3223
|
+
}
|
|
3224
|
+
const childBindingProjectionIds = childEffectIds.map(childEffectId => {
|
|
3225
|
+
const child = decisionChildEffects[childEffectId]
|
|
3226
|
+
if (child === undefined) throw new Error('parseman: decision effect lost its child effect')
|
|
3227
|
+
const edge = bindingEdges[child.childBindingEdgeId]
|
|
3228
|
+
if (edge?.projectionId === undefined) {
|
|
3229
|
+
throw new Error('parseman: decision effect lost its child binding projection')
|
|
3230
|
+
}
|
|
3231
|
+
return edge.projectionId
|
|
3232
|
+
})
|
|
3233
|
+
let readerMask = ALL_BINDING_READERS
|
|
3234
|
+
let variantMask = ALL_BINDING_VARIANTS
|
|
3235
|
+
for (const projectionId of childBindingProjectionIds) {
|
|
3236
|
+
const projection = bindingProjections[projectionId]
|
|
3237
|
+
if (projection === undefined) throw new Error('parseman: decision binding projection disappeared')
|
|
3238
|
+
readerMask &= projection.readerMask
|
|
3239
|
+
variantMask &= projection.variantMask
|
|
3240
|
+
}
|
|
3241
|
+
if (childBindingProjectionIds.length === 0) {
|
|
3242
|
+
readerMask = 0
|
|
3243
|
+
variantMask = 0
|
|
3244
|
+
}
|
|
3245
|
+
if (loweringGap === undefined
|
|
3246
|
+
&& (readerMask !== ALL_BINDING_READERS || variantMask !== ALL_BINDING_VARIANTS)) {
|
|
3247
|
+
loweringGap = 'decision child binding projection is incomplete'
|
|
3248
|
+
}
|
|
3249
|
+
const authority = finalDecisionAuthorities[authorityId]
|
|
3250
|
+
if (authority === undefined) throw new Error('parseman: decision effect lost its final authority')
|
|
3251
|
+
const digestWords = [
|
|
3252
|
+
decision.siteId, authorityId,
|
|
3253
|
+
authority.atom === 'choice' ? 0 : 1,
|
|
3254
|
+
DECISION_REFERENCE_TEMPLATE_ID, DECISION_CAPTURED_TEMPLATE_ID, DECISION_NAMED_TEMPLATE_ID,
|
|
3255
|
+
readerMask, variantMask, childEffectIds.length,
|
|
3256
|
+
...(decision.routeUsesRouted ?? []).map(value => value ? 1 : 0),
|
|
3257
|
+
...childEffectIds.flatMap((childEffectId, index) => {
|
|
3258
|
+
const child = decisionChildEffects[childEffectId]!
|
|
3259
|
+
const projection = bindingProjections[childBindingProjectionIds[index]!]!
|
|
3260
|
+
return [
|
|
3261
|
+
child.armId, child.expectedAuthorityId,
|
|
3262
|
+
child.gateCallbackId ?? -1, child.gateSourceId ?? -1, child.autoNotAuthorityId ?? -1,
|
|
3263
|
+
projection.id, projection.semanticDigest,
|
|
3264
|
+
]
|
|
3265
|
+
}),
|
|
3266
|
+
]
|
|
3267
|
+
decisionEffects.push({
|
|
3268
|
+
id: effectId, decisionSiteId: decision.siteId, finalDecisionAuthorityId: authorityId,
|
|
3269
|
+
phase: loweringGap === undefined ? COMPLETE_PHASE
|
|
3270
|
+
: phaseGap(COMPLETE_CAPABILITY, loweringGap),
|
|
3271
|
+
childEffectIds, childBindingProjectionIds,
|
|
3272
|
+
referenceTemplateId: DECISION_REFERENCE_TEMPLATE_ID,
|
|
3273
|
+
capturedTemplateId: DECISION_CAPTURED_TEMPLATE_ID,
|
|
3274
|
+
namedTemplateId: DECISION_NAMED_TEMPLATE_ID,
|
|
3275
|
+
readerMask, variantMask,
|
|
3276
|
+
semanticDigest: bindingDigest(digestWords, authority.atom, String(decision.siteId)),
|
|
3277
|
+
pendingRollbackPolicyId: 0, commitPolicyId: 0,
|
|
3278
|
+
})
|
|
3279
|
+
}
|
|
3280
|
+
return {
|
|
3281
|
+
finalDecisionAuthorities, decisionEffects, decisionChildEffects,
|
|
3282
|
+
choiceClassAuthorities, choiceExecutionOrders, classifySpellingMaps,
|
|
3283
|
+
classifyProjectionPlans, decisionExpectedAuthorities,
|
|
3284
|
+
decisionCallbackAuthorities,
|
|
3285
|
+
decisionSourceAuthorities: [...sourceIds.entries()].map(([source, id]) => ({ id, source }))
|
|
3286
|
+
.sort((a, b) => a.id - b.id),
|
|
3287
|
+
decisionAutoNotAuthorities: [...autoNotIds.entries()].map(([key, id]) => ({
|
|
3288
|
+
id, checks: JSON.parse(key) as AutoNotCheck[],
|
|
3289
|
+
})).sort((a, b) => a.id - b.id),
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
|
|
3293
|
+
function lexicalCapabilityInventory(
|
|
3294
|
+
roots: ReadonlyArray<Combinator<unknown>>,
|
|
3295
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
3296
|
+
): {
|
|
3297
|
+
capabilities: LexicalCapabilitySite[]
|
|
3298
|
+
languages: LexicalCapabilityLanguage[]
|
|
3299
|
+
bindingEdges: LexicalBindingEdge[]
|
|
3300
|
+
bindingProjections: LexicalBindingProjection[]
|
|
3301
|
+
terminalProjections: LexicalTerminalProjection[]
|
|
3302
|
+
decisionFamilies: LexicalDecisionFamily[]
|
|
3303
|
+
decisionOutcomes: LexicalDecisionOutcome[]
|
|
3304
|
+
decisions: LexicalDecisionSite[]
|
|
3305
|
+
transitionDiagnostics: LexicalTransitionDiagnosticPlan[]
|
|
3306
|
+
controlPlans: LexicalControlPlan[]
|
|
3307
|
+
boundaryPlans: LexicalBoundaryPlan[]
|
|
3308
|
+
materializationPlans: LexicalMaterializationPlan[]
|
|
3309
|
+
grammarWrapperSpecs: LexicalGrammarWrapperSpec[]
|
|
3310
|
+
grammarCaptureTriviaKinds: string[][]
|
|
3311
|
+
boundaryTopologies: LexicalBoundaryTopology[]
|
|
3312
|
+
contextSnapshots: LexicalContextSnapshot[]
|
|
3313
|
+
ownSkipPlans: LexicalOwnSkipPlan[]
|
|
3314
|
+
scanConsumerPolicies: LexicalScanConsumerPolicy[]
|
|
3315
|
+
decisionEffectPlan: LexicalDecisionEffectInventory
|
|
3316
|
+
} {
|
|
3317
|
+
const inventory = lexicalCapabilityCandidates(roots, resolve)
|
|
3318
|
+
const occurrenceContexts = new Map<Combinator<unknown>, Set<string>>()
|
|
3319
|
+
for (const candidate of inventory.candidates) {
|
|
3320
|
+
let contexts = occurrenceContexts.get(candidate.parser)
|
|
3321
|
+
if (contexts === undefined) occurrenceContexts.set(candidate.parser, contexts = new Set())
|
|
3322
|
+
contexts.add(candidate.contextKey)
|
|
3323
|
+
}
|
|
3324
|
+
const isSeparateOccurrence = (parser: Combinator<unknown>, contextKey: string): boolean =>
|
|
3325
|
+
occurrenceContexts.get(parser)?.has(contextKey) === true
|
|
3326
|
+
const scanConsumersBySite = new Map<number, ScanConsumerCandidate[]>()
|
|
3327
|
+
for (let siteId = 0; siteId < inventory.candidates.length; siteId++) {
|
|
3328
|
+
const candidate = inventory.candidates[siteId]!
|
|
3329
|
+
const consumers = ownedScanConsumers(
|
|
3330
|
+
candidate.parser, candidate.context, inventory.contextKeyOf, isSeparateOccurrence, resolve,
|
|
3331
|
+
)
|
|
3332
|
+
if (consumers.length === 0) continue
|
|
3333
|
+
scanConsumersBySite.set(siteId, consumers)
|
|
3334
|
+
for (const consumer of consumers) {
|
|
3335
|
+
if (!inventory.contexts.has(consumer.contextKey)) {
|
|
3336
|
+
inventory.contexts.set(consumer.contextKey, consumer.context)
|
|
3337
|
+
}
|
|
3338
|
+
}
|
|
3339
|
+
}
|
|
3340
|
+
const contextSnapshots = [...inventory.contexts.entries()]
|
|
3341
|
+
.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)
|
|
3342
|
+
.map(([key, context], id): LexicalContextSnapshot => ({
|
|
3343
|
+
id, key,
|
|
3344
|
+
hasTrivia: context.trivia !== undefined,
|
|
3345
|
+
hasScanSkip: context.scanSkip.length > 0,
|
|
3346
|
+
trackLines: context.trackLines,
|
|
3347
|
+
captureTrivia: context.captureTrivia,
|
|
3348
|
+
opaqueRootCapture: context.opaqueRootCapture,
|
|
3349
|
+
dynamicState: context.dynamicState,
|
|
3350
|
+
}))
|
|
3351
|
+
const contextSnapshotIdByKey = new Map(contextSnapshots.map(snapshot => [snapshot.key, snapshot.id]))
|
|
3352
|
+
const decisionInventory = lexicalDecisionInventory(
|
|
3353
|
+
inventory.candidates, contextSnapshotIdByKey, resolve,
|
|
3354
|
+
)
|
|
3355
|
+
const transitionDiagnostics: LexicalTransitionDiagnosticPlan[] = []
|
|
3356
|
+
const transitionDiagnosticIdByKey = new Map<string, number>()
|
|
3357
|
+
const internTransitionDiagnostics = (
|
|
3358
|
+
recognizerKey: string,
|
|
3359
|
+
session: LexicalNormalizeSession,
|
|
3360
|
+
): number => {
|
|
3361
|
+
const body = {
|
|
3362
|
+
stateCount: session.nextState, expected: session.expected, events: session.events,
|
|
3363
|
+
}
|
|
3364
|
+
// State ids are local to the canonical recognizer transition graph. Equal
|
|
3365
|
+
// event rows over a different graph are not one plan merely by coincidence.
|
|
3366
|
+
const key = `${recognizerKey}\u0000${JSON.stringify(body)}`
|
|
3367
|
+
const prior = transitionDiagnosticIdByKey.get(key)
|
|
3368
|
+
if (prior !== undefined) return prior
|
|
3369
|
+
const id = transitionDiagnostics.length
|
|
3370
|
+
transitionDiagnosticIdByKey.set(key, id)
|
|
3371
|
+
transitionDiagnostics.push({ id, ...body })
|
|
3372
|
+
return id
|
|
3373
|
+
}
|
|
3374
|
+
const controlPlans: LexicalControlPlan[] = []
|
|
3375
|
+
const controlPlanIdByKey = new Map<string, number>()
|
|
3376
|
+
const internControlPlan = (session: LexicalNormalizeSession): number => {
|
|
3377
|
+
const key = JSON.stringify(session.controls)
|
|
3378
|
+
const prior = controlPlanIdByKey.get(key)
|
|
3379
|
+
if (prior !== undefined) return prior
|
|
3380
|
+
const id = controlPlans.length
|
|
3381
|
+
controlPlanIdByKey.set(key, id)
|
|
3382
|
+
controlPlans.push({ id, controls: session.controls })
|
|
3383
|
+
return id
|
|
3384
|
+
}
|
|
3385
|
+
const grammarWrapperSpecs: LexicalGrammarWrapperSpec[] = []
|
|
3386
|
+
const grammarSpecIdByKey = new Map<string, number>()
|
|
3387
|
+
const grammarSourceOperationIds = new Map<Combinator<unknown>, number>()
|
|
3388
|
+
const triviaBindingIds = new Map<Combinator<unknown>, number>()
|
|
3389
|
+
const grammarCaptureTriviaKinds: string[][] = []
|
|
3390
|
+
const captureKindsIdByKey = new Map<string, number>()
|
|
3391
|
+
const boundaryTopologies: LexicalBoundaryTopology[] = []
|
|
3392
|
+
const topologyIdByKey = new Map<string, number>()
|
|
3393
|
+
const identityId = (map: Map<Combinator<unknown>, number>, parser: Combinator<unknown>): number => {
|
|
3394
|
+
const prior = map.get(parser)
|
|
3395
|
+
if (prior !== undefined) return prior
|
|
3396
|
+
const id = map.size
|
|
3397
|
+
map.set(parser, id)
|
|
3398
|
+
return id
|
|
3399
|
+
}
|
|
3400
|
+
const captureKindsId = (values: readonly string[] | undefined): number | undefined => {
|
|
3401
|
+
if (values === undefined) return undefined
|
|
3402
|
+
const key = JSON.stringify(values)
|
|
3403
|
+
const prior = captureKindsIdByKey.get(key)
|
|
3404
|
+
if (prior !== undefined) return prior
|
|
3405
|
+
const id = grammarCaptureTriviaKinds.length
|
|
3406
|
+
captureKindsIdByKey.set(key, id)
|
|
3407
|
+
grammarCaptureTriviaKinds.push([...values])
|
|
3408
|
+
return id
|
|
3409
|
+
}
|
|
3410
|
+
const internGrammarWrapper = (
|
|
3411
|
+
parser: Combinator<unknown>,
|
|
3412
|
+
captureTriviaKinds: readonly string[] | undefined,
|
|
3413
|
+
): number | undefined => {
|
|
3414
|
+
const def = parser._def
|
|
3415
|
+
if (def.tag !== 'grammar') return undefined
|
|
3416
|
+
if (def.constructionTrackLines === undefined) return undefined
|
|
3417
|
+
const sourceOperationId = identityId(grammarSourceOperationIds, parser)
|
|
3418
|
+
const triviaBindingId = def.triviaParser === undefined
|
|
3419
|
+
? undefined : identityId(triviaBindingIds, def.triviaParser)
|
|
3420
|
+
const captureTriviaKindsId = captureKindsId(captureTriviaKinds)
|
|
3421
|
+
const body = {
|
|
3422
|
+
sourceOperationId,
|
|
3423
|
+
clearTrivia: def.clearTrivia === true,
|
|
3424
|
+
...(triviaBindingId === undefined ? {} : { triviaBindingId }),
|
|
3425
|
+
trackLines: def.constructionTrackLines,
|
|
3426
|
+
captureTrivia: def.captureTrivia === true,
|
|
3427
|
+
...(captureTriviaKindsId === undefined ? {} : { captureTriviaKindsId }),
|
|
3428
|
+
rootCapture: def.rootCapture === 'opaque' ? 'opaque' as const : 'inherit' as const,
|
|
3429
|
+
clonePolicy: 'spread-existing-or-create-canonical' as const,
|
|
3430
|
+
postChildPolicy: 'line-propagate-then-annotate-or-return' as const,
|
|
3431
|
+
}
|
|
3432
|
+
const key = JSON.stringify(body)
|
|
3433
|
+
const prior = grammarSpecIdByKey.get(key)
|
|
3434
|
+
if (prior !== undefined) return prior
|
|
3435
|
+
const id = grammarWrapperSpecs.length
|
|
3436
|
+
grammarSpecIdByKey.set(key, id)
|
|
3437
|
+
grammarWrapperSpecs.push({ id, ...body })
|
|
3438
|
+
return id
|
|
3439
|
+
}
|
|
3440
|
+
const internBoundaryTopology = (
|
|
3441
|
+
session: LexicalNormalizeSession,
|
|
3442
|
+
): { readonly id: number } | { readonly gap: string } => {
|
|
3443
|
+
// Validate the entire overlay before interning any source-operation spec.
|
|
3444
|
+
// A declined late frame must not leave unreachable compiler metadata behind.
|
|
3445
|
+
const grammarCaptureKindsByFrame = new Map<number, readonly string[] | undefined>()
|
|
3446
|
+
for (let frameIndex = 0; frameIndex < session.wrapperFrames.length; frameIndex++) {
|
|
3447
|
+
const pending = session.wrapperFrames[frameIndex]!
|
|
3448
|
+
const control = session.controls[pending.controlId]
|
|
3449
|
+
if (pending.stateStart < 0 || pending.stateEnd < pending.stateStart
|
|
3450
|
+
|| pending.stateEnd > session.nextState
|
|
3451
|
+
|| pending.id !== frameIndex || control === undefined
|
|
3452
|
+
|| control.id !== pending.controlId
|
|
3453
|
+
|| control.kind !== pending.kind
|
|
3454
|
+
|| control.stateStart !== pending.stateStart
|
|
3455
|
+
|| control.stateEnd !== pending.stateEnd) {
|
|
3456
|
+
return { gap: 'token boundary overlay has an invalid state interval' }
|
|
3457
|
+
}
|
|
3458
|
+
if (pending.parentFrameId !== undefined) {
|
|
3459
|
+
const parent = session.wrapperFrames[pending.parentFrameId]
|
|
3460
|
+
if (parent === undefined || pending.parentFrameId >= pending.id
|
|
3461
|
+
|| pending.stateStart < parent.stateStart || pending.stateEnd > parent.stateEnd) {
|
|
3462
|
+
return { gap: 'token boundary overlay is not well nested' }
|
|
3463
|
+
}
|
|
3464
|
+
}
|
|
3465
|
+
if (pending.kind === 'grammar') {
|
|
3466
|
+
const def = pending.parser._def
|
|
3467
|
+
if (def.tag !== 'grammar' || def.constructionTrackLines === undefined) {
|
|
3468
|
+
return { gap: 'grammar wrapper lacks exact construction-time source-operation metadata' }
|
|
3469
|
+
}
|
|
3470
|
+
try {
|
|
3471
|
+
grammarCaptureKindsByFrame.set(
|
|
3472
|
+
pending.id,
|
|
3473
|
+
def.constructionCaptureTriviaKinds === undefined
|
|
3474
|
+
? undefined : [...def.constructionCaptureTriviaKinds],
|
|
3475
|
+
)
|
|
3476
|
+
} catch {
|
|
3477
|
+
return { gap: 'grammar wrapper capture-trivia policy cannot be snapshotted safely' }
|
|
3478
|
+
}
|
|
3479
|
+
}
|
|
3480
|
+
}
|
|
3481
|
+
for (let i = 0; i < session.wrapperFrames.length; i++) {
|
|
3482
|
+
for (let j = i + 1; j < session.wrapperFrames.length; j++) {
|
|
3483
|
+
const a = session.wrapperFrames[i]!
|
|
3484
|
+
const b = session.wrapperFrames[j]!
|
|
3485
|
+
const overlap = a.stateStart < b.stateEnd && b.stateStart < a.stateEnd
|
|
3486
|
+
const nested = (a.stateStart <= b.stateStart && b.stateEnd <= a.stateEnd)
|
|
3487
|
+
|| (b.stateStart <= a.stateStart && a.stateEnd <= b.stateEnd)
|
|
3488
|
+
if (overlap && !nested) return { gap: 'token boundary overlay has crossed state intervals' }
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
const outer = session.wrapperFrames[0]
|
|
3492
|
+
if (outer?.kind !== 'token' || outer.parentFrameId !== undefined
|
|
3493
|
+
|| outer.stateStart !== 0 || outer.stateEnd !== session.nextState) {
|
|
3494
|
+
return { gap: 'token boundary overlay lost its outer authored token frame' }
|
|
3495
|
+
}
|
|
3496
|
+
|
|
3497
|
+
const frames: LexicalWrapperFrame[] = []
|
|
3498
|
+
for (const pending of session.wrapperFrames) {
|
|
3499
|
+
if (pending.kind === 'token') frames.push({
|
|
3500
|
+
id: pending.id,
|
|
3501
|
+
...(pending.parentFrameId === undefined ? {} : { parentFrameId: pending.parentFrameId }),
|
|
3502
|
+
controlId: pending.controlId,
|
|
3503
|
+
stateStart: pending.stateStart,
|
|
3504
|
+
stateEnd: pending.stateEnd,
|
|
3505
|
+
kind: 'token', boundaryPlanId: 0, materializationPlanId: 0,
|
|
3506
|
+
})
|
|
3507
|
+
else {
|
|
3508
|
+
const wrapperSpecId = internGrammarWrapper(
|
|
3509
|
+
pending.parser, grammarCaptureKindsByFrame.get(pending.id),
|
|
3510
|
+
)
|
|
3511
|
+
if (wrapperSpecId === undefined) throw new Error('parseman: validated grammar policy disappeared')
|
|
3512
|
+
frames.push({
|
|
3513
|
+
id: pending.id,
|
|
3514
|
+
...(pending.parentFrameId === undefined ? {} : { parentFrameId: pending.parentFrameId }),
|
|
3515
|
+
controlId: pending.controlId,
|
|
3516
|
+
stateStart: pending.stateStart,
|
|
3517
|
+
stateEnd: pending.stateEnd,
|
|
3518
|
+
kind: 'grammar', wrapperSpecId,
|
|
3519
|
+
})
|
|
3520
|
+
}
|
|
3521
|
+
}
|
|
3522
|
+
const key = JSON.stringify(frames)
|
|
3523
|
+
const prior = topologyIdByKey.get(key)
|
|
3524
|
+
if (prior !== undefined) return { id: prior }
|
|
3525
|
+
const id = boundaryTopologies.length
|
|
3526
|
+
topologyIdByKey.set(key, id)
|
|
3527
|
+
boundaryTopologies.push({ id, frames })
|
|
3528
|
+
return { id }
|
|
3529
|
+
}
|
|
3530
|
+
const ownSkipPlans: LexicalOwnSkipPlan[] = []
|
|
3531
|
+
const ownSkipPlanIdByKey = new Map<string, number>()
|
|
3532
|
+
const internOwnSkipPlan = (entries: readonly Combinator<unknown>[]): number | undefined => {
|
|
3533
|
+
if (entries.length === 0) return undefined
|
|
3534
|
+
const represented = entries.map((entry): LexicalOwnSkipEntry => {
|
|
3535
|
+
const normalized = normalizeBalancedLexical(entry, resolve)
|
|
3536
|
+
?? normalizeLexical(entry, resolve, new Set())
|
|
3537
|
+
if ('refusal' in normalized) return {
|
|
3538
|
+
status: gap(`ownSkip normalization: ${normalized.refusal}`),
|
|
3539
|
+
}
|
|
3540
|
+
return {
|
|
3541
|
+
semanticKey: JSON.stringify(normalized.ir), ir: normalized.ir, status: COMPLETE_CAPABILITY,
|
|
3542
|
+
}
|
|
3543
|
+
})
|
|
3544
|
+
const key = JSON.stringify(represented)
|
|
3545
|
+
const prior = ownSkipPlanIdByKey.get(key)
|
|
3546
|
+
if (prior !== undefined) return prior
|
|
3547
|
+
const id = ownSkipPlans.length
|
|
3548
|
+
ownSkipPlanIdByKey.set(key, id)
|
|
3549
|
+
ownSkipPlans.push({ id, entries: represented })
|
|
3550
|
+
return id
|
|
3551
|
+
}
|
|
3552
|
+
const scanConsumerPolicies: LexicalScanConsumerPolicy[] = []
|
|
3553
|
+
const scanPolicyIdsBySite = new Map<number, number[]>()
|
|
3554
|
+
for (let siteId = 0; siteId < inventory.candidates.length; siteId++) {
|
|
3555
|
+
const candidate = inventory.candidates[siteId]!
|
|
3556
|
+
for (const consumer of scanConsumersBySite.get(siteId) ?? []) {
|
|
3557
|
+
const contextSnapshotId = contextSnapshotIdByKey.get(consumer.contextKey)
|
|
3558
|
+
if (contextSnapshotId === undefined) throw new Error('parseman: scan consumer lost its context snapshot')
|
|
3559
|
+
const ownSkipPlanId = internOwnSkipPlan(consumer.ownSkip)
|
|
3560
|
+
const ownSkipPlan = ownSkipPlanId === undefined ? undefined : ownSkipPlans[ownSkipPlanId]
|
|
3561
|
+
const status = ownSkipPlan?.entries.find(entry => entry.status.kind !== 'complete')?.status
|
|
3562
|
+
?? COMPLETE_CAPABILITY
|
|
3563
|
+
const id = scanConsumerPolicies.length
|
|
3564
|
+
const common = {
|
|
3565
|
+
id, siteId, path: `${candidate.path}${consumer.path}`,
|
|
3566
|
+
contextSnapshotId, raw: consumer.raw,
|
|
3567
|
+
...(ownSkipPlanId === undefined ? {} : { ownSkipPlanId }),
|
|
3568
|
+
lookup: 'dynamic-parse-property-every-skip-test' as const,
|
|
3569
|
+
pendingReuse: 'forbidden' as const,
|
|
3570
|
+
status,
|
|
3571
|
+
}
|
|
3572
|
+
scanConsumerPolicies.push(consumer.kind === 'scanTo' ? {
|
|
3573
|
+
...common,
|
|
3574
|
+
kind: 'scanTo',
|
|
3575
|
+
ambient: consumer.raw ? 'none-raw' : 'enumerate-trivia-then-scanSkip-per-parse-attempt',
|
|
3576
|
+
context: 'detached-per-attempt-state-errors-only',
|
|
3577
|
+
} : {
|
|
3578
|
+
...common,
|
|
3579
|
+
kind: 'balanced',
|
|
3580
|
+
ambient: consumer.raw ? 'none-raw' : 'array-identity-interior-cache',
|
|
3581
|
+
cache: 'lookup-every-attempt-enumerate-on-miss',
|
|
3582
|
+
context: 'token-cleared-original',
|
|
3583
|
+
})
|
|
3584
|
+
let ids = scanPolicyIdsBySite.get(siteId)
|
|
3585
|
+
if (ids === undefined) scanPolicyIdsBySite.set(siteId, ids = [])
|
|
3586
|
+
ids.push(id)
|
|
3587
|
+
}
|
|
3588
|
+
}
|
|
3589
|
+
|
|
3590
|
+
const pending = inventory.candidates.map((candidate, id) => {
|
|
3591
|
+
const contextSnapshotId = contextSnapshotIdByKey.get(candidate.contextKey)
|
|
3592
|
+
if (contextSnapshotId === undefined) throw new Error('parseman: capability occurrence lost its context snapshot')
|
|
3593
|
+
const scanConsumerPolicyIds = scanPolicyIdsBySite.get(id) ?? []
|
|
3594
|
+
if (candidate.atom === 'terminal') {
|
|
3595
|
+
const key = keyOf(candidate.parser._def)
|
|
3596
|
+
if (key === undefined) throw new Error('parseman: lexical terminal capability lost its semantic key')
|
|
3597
|
+
const obligations = terminalObligations()
|
|
3598
|
+
return {
|
|
3599
|
+
id, bodyId: candidate.bodyId,
|
|
3600
|
+
path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
|
|
3601
|
+
recognitionContextKey: candidate.recognitionContextKey,
|
|
3602
|
+
context: candidate.context,
|
|
3603
|
+
recognitionContext: candidate.parser._def.tag === 'token'
|
|
3604
|
+
? { ...candidate.context, trivia: undefined, captureTrivia: false }
|
|
3605
|
+
: candidate.context,
|
|
3606
|
+
semanticKey: key, atom: candidate.atom,
|
|
3607
|
+
parser: candidate.parser, obligations, scanConsumerPolicyIds,
|
|
3608
|
+
status: derivedCapabilityStatus(obligations),
|
|
3609
|
+
}
|
|
3610
|
+
}
|
|
3611
|
+
const def = candidate.parser._def
|
|
3612
|
+
if (candidate.atom === 'choice') {
|
|
3613
|
+
if (def.tag !== 'choice') throw new Error('parseman: lexical choice capability lost its boundary')
|
|
3614
|
+
const obligations = decisionObligations(decisionInventory.recognitionBySite.get(id)
|
|
3615
|
+
?? gap('choice outcome recognition record is missing'))
|
|
3616
|
+
return {
|
|
3617
|
+
id, bodyId: candidate.bodyId, path: candidate.path, contextSnapshotId,
|
|
3618
|
+
contextKey: candidate.contextKey,
|
|
3619
|
+
recognitionContextKey: candidate.recognitionContextKey,
|
|
3620
|
+
context: candidate.context,
|
|
3621
|
+
recognitionContext: candidate.context,
|
|
3622
|
+
semanticKey: `C\u0000${def.parsers.length}\u0000${def.strategy.tag}`,
|
|
3623
|
+
atom: candidate.atom, parser: candidate.parser, obligations, scanConsumerPolicyIds,
|
|
3624
|
+
status: derivedCapabilityStatus(obligations),
|
|
3625
|
+
}
|
|
3626
|
+
}
|
|
3627
|
+
if (candidate.atom === 'dispatch') {
|
|
3628
|
+
if (def.tag !== 'dispatch') throw new Error('parseman: lexical dispatch capability lost its boundary')
|
|
3629
|
+
const obligations = decisionObligations(decisionInventory.recognitionBySite.get(id)
|
|
3630
|
+
?? gap('dispatch outcome recognition record is missing'))
|
|
3631
|
+
return {
|
|
3632
|
+
id, bodyId: candidate.bodyId, path: candidate.path, contextSnapshotId,
|
|
3633
|
+
contextKey: candidate.contextKey,
|
|
3634
|
+
recognitionContextKey: candidate.recognitionContextKey,
|
|
3635
|
+
context: candidate.context,
|
|
3636
|
+
recognitionContext: candidate.context,
|
|
3637
|
+
semanticKey: `D\u0000${def.cases.length}\u0000${def.matchers?.length ?? 0}\u0000${def.otherwise === undefined ? 0 : 1}`,
|
|
3638
|
+
atom: candidate.atom, parser: candidate.parser, obligations, scanConsumerPolicyIds,
|
|
3639
|
+
status: derivedCapabilityStatus(obligations),
|
|
3640
|
+
}
|
|
3641
|
+
}
|
|
3642
|
+
if (def.tag !== 'token') throw new Error('parseman: lexical token capability lost its boundary')
|
|
3643
|
+
const session = newNormalizeSession()
|
|
3644
|
+
const normalized = normalizeBalancedLexical(
|
|
3645
|
+
candidate.parser, resolve, session,
|
|
3646
|
+
) ?? normalizeLexical(candidate.parser, resolve, new Set(), session)
|
|
3647
|
+
if ('refusal' in normalized) {
|
|
3648
|
+
const obligations = tokenObligations(gap(`token normalization: ${normalized.refusal}`))
|
|
3649
|
+
return {
|
|
3650
|
+
id, bodyId: candidate.bodyId,
|
|
3651
|
+
path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
|
|
3652
|
+
recognitionContextKey: candidate.recognitionContextKey,
|
|
3653
|
+
context: candidate.context,
|
|
3654
|
+
recognitionContext: { ...candidate.context, trivia: undefined, captureTrivia: false },
|
|
3655
|
+
semanticKey: `T\u0000GAP\u0000${normalized.refusal}`,
|
|
3656
|
+
atom: candidate.atom, parser: candidate.parser, obligations, scanConsumerPolicyIds,
|
|
3657
|
+
status: derivedCapabilityStatus(obligations),
|
|
3658
|
+
}
|
|
3659
|
+
}
|
|
3660
|
+
const semanticKey = `T\u0000${JSON.stringify(normalized.ir)}`
|
|
3661
|
+
if (matchesEmpty(def.parser)) {
|
|
3662
|
+
const obligations = tokenObligations({
|
|
3663
|
+
kind: 'impossible',
|
|
3664
|
+
proof: 'an atomic source token must consume positive width but this token body is nullable',
|
|
3665
|
+
})
|
|
3666
|
+
return {
|
|
3667
|
+
id, bodyId: candidate.bodyId,
|
|
3668
|
+
path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
|
|
3669
|
+
recognitionContextKey: candidate.recognitionContextKey,
|
|
3670
|
+
context: candidate.context,
|
|
3671
|
+
recognitionContext: { ...candidate.context, trivia: undefined, captureTrivia: false },
|
|
3672
|
+
semanticKey, atom: candidate.atom, parser: candidate.parser, scanConsumerPolicyIds,
|
|
3673
|
+
obligations, status: derivedCapabilityStatus(obligations),
|
|
3674
|
+
}
|
|
3675
|
+
}
|
|
3676
|
+
const diagnosticPlanId = internTransitionDiagnostics(semanticKey, session)
|
|
3677
|
+
const controlPlanId = internControlPlan(session)
|
|
3678
|
+
const topology = internBoundaryTopology(session)
|
|
3679
|
+
const topologyRepresentation = 'id' in topology
|
|
3680
|
+
? COMPLETE_CAPABILITY : gap(topology.gap)
|
|
3681
|
+
const obligations = directExecutableTokenBody(candidate.parser, resolve) === undefined
|
|
3682
|
+
? tokenObligations(COMPLETE_CAPABILITY, {
|
|
3683
|
+
diagnostics: true,
|
|
3684
|
+
boundaryPlan: topologyRepresentation,
|
|
3685
|
+
materializationPlan: topologyRepresentation,
|
|
3686
|
+
})
|
|
3687
|
+
: executableLexBodyObligations()
|
|
3688
|
+
return {
|
|
3689
|
+
id, bodyId: candidate.bodyId,
|
|
3690
|
+
path: candidate.path, contextKey: candidate.contextKey, contextSnapshotId,
|
|
3691
|
+
recognitionContextKey: candidate.recognitionContextKey,
|
|
3692
|
+
context: candidate.context,
|
|
3693
|
+
recognitionContext: { ...candidate.context, trivia: undefined, captureTrivia: false },
|
|
3694
|
+
semanticKey, atom: candidate.atom, parser: candidate.parser, diagnosticPlanId,
|
|
3695
|
+
scanConsumerPolicyIds,
|
|
3696
|
+
controlPlanId,
|
|
3697
|
+
...('id' in topology ? { boundaryTopologyId: topology.id } : {}),
|
|
3698
|
+
obligations, status: derivedCapabilityStatus(obligations),
|
|
3699
|
+
}
|
|
3700
|
+
})
|
|
3701
|
+
const languageKeys = [...new Set(pending.map(site => `${site.atom}\u0000${site.semanticKey}`))].sort()
|
|
3702
|
+
const languageIdByKey = new Map(languageKeys.map((key, id) => [key, id]))
|
|
3703
|
+
const languages = languageKeys.map((key, id): LexicalCapabilityLanguage => {
|
|
3704
|
+
const split = key.indexOf('\u0000')
|
|
3705
|
+
return {
|
|
3706
|
+
id,
|
|
3707
|
+
atom: key.slice(0, split) as LexicalCapabilitySite['atom'],
|
|
3708
|
+
semanticKey: key.slice(split + 1),
|
|
3709
|
+
}
|
|
3710
|
+
})
|
|
3711
|
+
const preliminaryCapabilities = pending.map(site => ({
|
|
3712
|
+
...site,
|
|
3713
|
+
languageId: languageIdByKey.get(`${site.atom}\u0000${site.semanticKey}`)!,
|
|
3714
|
+
}))
|
|
3715
|
+
const bindingProjections: LexicalBindingProjection[] = []
|
|
3716
|
+
const bindingEdges = inventory.bindingEdges.map((edge, id): LexicalBindingEdge => {
|
|
3717
|
+
const projection = fixedBindingProjection(edge, id, bindingProjections.length)
|
|
3718
|
+
bindingProjections.push(projection)
|
|
3719
|
+
const {
|
|
3720
|
+
parentBodyId: _parentBodyId, childBodyId: _childBodyId,
|
|
3721
|
+
childOrdinal: _childOrdinal, ...publicEdge
|
|
3722
|
+
} = edge
|
|
3723
|
+
// This record is executable evidence rather than a status assertion: its
|
|
3724
|
+
// numeric reference operand, captured/CSP slot+template, named symbol+template,
|
|
3725
|
+
// and common digest describe all readers before pricing. Scanner/recovery
|
|
3726
|
+
// edges remain GAP until their missing reader templates exist.
|
|
3727
|
+
const complete = projection.readerMask === ALL_BINDING_READERS
|
|
3728
|
+
&& projection.variantMask === ALL_BINDING_VARIANTS
|
|
3729
|
+
return {
|
|
3730
|
+
id, ...publicEdge, projectionId: projection.id,
|
|
3731
|
+
contextSnapshotId: contextSnapshotIdByKey.get(edge.contextKey)
|
|
3732
|
+
?? (() => { throw new Error('parseman: binding edge lost its context snapshot') })(),
|
|
3733
|
+
status: complete ? COMPLETE_CAPABILITY : gap(FIXED_TUPLE_BINDING_GAP),
|
|
3734
|
+
}
|
|
3735
|
+
})
|
|
3736
|
+
const terminalProjections = preliminaryCapabilities
|
|
3737
|
+
.filter(site => site.atom === 'terminal')
|
|
3738
|
+
.map((site, id): LexicalTerminalProjection => {
|
|
3739
|
+
const incomingBindingProjectionIds: number[] = []
|
|
3740
|
+
let readerMask = ALL_BINDING_READERS
|
|
3741
|
+
let variantMask = ALL_BINDING_VARIANTS
|
|
3742
|
+
for (let edgeId = 0; edgeId < inventory.bindingEdges.length; edgeId++) {
|
|
3743
|
+
const edge = inventory.bindingEdges[edgeId]!
|
|
3744
|
+
if (edge.childBodyId !== site.bodyId || edge.childContextKey !== site.contextKey) continue
|
|
3745
|
+
const bindingEdge = bindingEdges[edgeId]!
|
|
3746
|
+
const projection = bindingProjections[bindingEdge.projectionId!]
|
|
3747
|
+
if (projection === undefined) throw new Error('parseman: terminal binding projection disappeared')
|
|
3748
|
+
incomingBindingProjectionIds.push(projection.id)
|
|
3749
|
+
readerMask &= projection.readerMask
|
|
3750
|
+
variantMask &= projection.variantMask
|
|
3751
|
+
}
|
|
3752
|
+
if (incomingBindingProjectionIds.length === 0) {
|
|
3753
|
+
readerMask = 0
|
|
3754
|
+
variantMask = 0
|
|
3755
|
+
}
|
|
3756
|
+
const complete = incomingBindingProjectionIds.length > 0
|
|
3757
|
+
&& readerMask === ALL_BINDING_READERS
|
|
3758
|
+
&& variantMask === ALL_BINDING_VARIANTS
|
|
3759
|
+
&& incomingBindingProjectionIds.every(projectionId =>
|
|
3760
|
+
bindingEdges[bindingProjections[projectionId]!.edgeId]!.status.kind === 'complete')
|
|
3761
|
+
const digestWords = [
|
|
3762
|
+
site.bodyId, site.languageId, site.bodyId,
|
|
3763
|
+
TERMINAL_CAPTURED_TEMPLATE_ID, site.bodyId,
|
|
3764
|
+
readerMask, variantMask, incomingBindingProjectionIds.length,
|
|
3765
|
+
...incomingBindingProjectionIds.flatMap(projectionId => {
|
|
3766
|
+
const projection = bindingProjections[projectionId]!
|
|
3767
|
+
return [projection.id, projection.semanticDigest]
|
|
3768
|
+
}),
|
|
3769
|
+
]
|
|
3770
|
+
return {
|
|
3771
|
+
id, siteId: site.id, bodyId: site.bodyId, languageId: site.languageId,
|
|
3772
|
+
referenceBodyId: site.bodyId,
|
|
3773
|
+
capturedTemplateId: TERMINAL_CAPTURED_TEMPLATE_ID,
|
|
3774
|
+
namedSymbolId: site.bodyId,
|
|
3775
|
+
incomingBindingProjectionIds,
|
|
3776
|
+
readerMask, variantMask,
|
|
3777
|
+
semanticDigest: bindingDigest(digestWords, 'terminal', site.semanticKey),
|
|
3778
|
+
status: complete ? COMPLETE_CAPABILITY : gap(FIXED_TUPLE_BINDING_GAP),
|
|
3779
|
+
}
|
|
3780
|
+
})
|
|
3781
|
+
const terminalProjectionBySiteId = new Map(terminalProjections.map(projection =>
|
|
3782
|
+
[projection.siteId, projection]))
|
|
3783
|
+
const terminalCapabilities = preliminaryCapabilities.map((site): LexicalCapabilitySite => {
|
|
3784
|
+
if (site.atom !== 'terminal') return site
|
|
3785
|
+
const projection = terminalProjectionBySiteId.get(site.id)
|
|
3786
|
+
if (projection === undefined) throw new Error('parseman: terminal capability lost its projection')
|
|
3787
|
+
const bindingAndReachability = projection.status.kind === 'complete'
|
|
3788
|
+
? COMPLETE_PHASE
|
|
3789
|
+
: phaseGap(projection.status, FIXED_TUPLE_BINDING_GAP)
|
|
3790
|
+
const obligations = { ...site.obligations, bindingAndReachability }
|
|
3791
|
+
return { ...site, obligations, status: derivedCapabilityStatus(obligations) }
|
|
3792
|
+
})
|
|
3793
|
+
const decisionEffects = decisionEffectInventory(
|
|
3794
|
+
inventory.candidates, decisionInventory.decisions, bindingEdges, bindingProjections, resolve,
|
|
3795
|
+
)
|
|
3796
|
+
const decisionEffectBySite = new Map(decisionEffects.decisionEffects.map(effect =>
|
|
3797
|
+
[effect.decisionSiteId, effect]))
|
|
3798
|
+
const capabilities = terminalCapabilities.map((site): LexicalCapabilitySite => {
|
|
3799
|
+
if (site.atom !== 'choice' && site.atom !== 'dispatch') return site
|
|
3800
|
+
const effect = decisionEffectBySite.get(site.id)
|
|
3801
|
+
if (effect === undefined) throw new Error('parseman: decision capability lost its effect projection')
|
|
3802
|
+
const obligations = executableDecisionObligations(
|
|
3803
|
+
site.obligations.recognition.representation,
|
|
3804
|
+
effect.phase,
|
|
3805
|
+
)
|
|
3806
|
+
return { ...site, obligations, status: derivedCapabilityStatus(obligations) }
|
|
3807
|
+
})
|
|
3808
|
+
return {
|
|
3809
|
+
capabilities, languages, bindingEdges, bindingProjections, terminalProjections,
|
|
3810
|
+
decisionFamilies: decisionInventory.families,
|
|
3811
|
+
decisionOutcomes: decisionInventory.outcomes,
|
|
3812
|
+
decisions: decisionInventory.decisions,
|
|
3813
|
+
decisionEffectPlan: decisionEffects,
|
|
3814
|
+
transitionDiagnostics,
|
|
3815
|
+
controlPlans,
|
|
3816
|
+
boundaryPlans: boundaryTopologies.length === 0
|
|
3817
|
+
? [] : [{ id: 0, kind: 'token-context-transaction' }],
|
|
3818
|
+
materializationPlans: boundaryTopologies.length === 0
|
|
3819
|
+
? [] : [{ id: 0, kind: 'token-source-range' }],
|
|
3820
|
+
grammarWrapperSpecs,
|
|
3821
|
+
grammarCaptureTriviaKinds,
|
|
3822
|
+
boundaryTopologies,
|
|
3823
|
+
contextSnapshots,
|
|
3824
|
+
ownSkipPlans,
|
|
3825
|
+
scanConsumerPolicies,
|
|
3826
|
+
}
|
|
3827
|
+
}
|
|
3828
|
+
|
|
3829
|
+
/** Re-enumerate the final graph so a dropped/filtered candidate fails closed. */
|
|
3830
|
+
export function assertLexicalCapabilityClosure(
|
|
3831
|
+
roots: ReadonlyArray<Combinator<unknown>>,
|
|
3832
|
+
alphabet: Pick<LexicalCapabilityInventory,
|
|
3833
|
+
'capabilities' | 'capabilityLanguages' | 'bindingEdges' | 'decisionFamilies'
|
|
3834
|
+
| 'decisionOutcomes' | 'decisions'> & Partial<Pick<LexicalCapabilityInventory,
|
|
3835
|
+
'bindingProjections' | 'terminalProjections' | 'decisionEffectPlan'
|
|
3836
|
+
| 'transitionDiagnostics' | 'boundaryPlans' | 'materializationPlans'
|
|
3837
|
+
| 'grammarWrapperSpecs' | 'grammarCaptureTriviaKinds' | 'boundaryTopologies'
|
|
3838
|
+
| 'controlPlans' | 'contextSnapshots' | 'ownSkipPlans' | 'scanConsumerPolicies'>>,
|
|
3839
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
3840
|
+
): void {
|
|
3841
|
+
const actual = lexicalCapabilityInventory(roots, resolve)
|
|
3842
|
+
const signature = (site: LexicalCapabilitySite): string =>
|
|
3843
|
+
`${site.id}\u0000${site.bodyId}\u0000${site.languageId}\u0000${site.path}\u0000${site.contextKey}\u0000${site.contextSnapshotId}\u0000${site.recognitionContextKey}\u0000${site.atom}\u0000${site.semanticKey}\u0000${site.diagnosticPlanId ?? -1}\u0000${site.controlPlanId ?? -1}\u0000${site.boundaryTopologyId ?? -1}\u0000${site.scanConsumerPolicyIds.join(',')}\u0000${JSON.stringify(site.obligations)}`
|
|
3844
|
+
const expectedKeys = actual.capabilities.map(signature)
|
|
3845
|
+
const suppliedKeys = alphabet.capabilities.map(signature)
|
|
3846
|
+
const expectedLanguages = actual.languages.map(language =>
|
|
3847
|
+
`${language.id}\u0000${language.atom}\u0000${language.semanticKey}`)
|
|
3848
|
+
const suppliedLanguages = alphabet.capabilityLanguages.map(language =>
|
|
3849
|
+
`${language.id}\u0000${language.atom}\u0000${language.semanticKey}`)
|
|
3850
|
+
const edgeSignature = (edge: LexicalBindingEdge): string =>
|
|
3851
|
+
`${edge.id}\u0000${edge.path}\u0000${edge.contextKey}\u0000${edge.childContextKey}\u0000${edge.contextSnapshotId}\u0000${edge.parentTag}\u0000${edge.childTag}\u0000${edge.projectionId ?? -1}\u0000${JSON.stringify(edge.status)}`
|
|
3852
|
+
const expectedEdges = actual.bindingEdges.map(edgeSignature)
|
|
3853
|
+
const suppliedEdges = alphabet.bindingEdges.map(edgeSignature)
|
|
3854
|
+
const expectedBindingProjections = actual.bindingProjections.map(plan => JSON.stringify(plan))
|
|
3855
|
+
const suppliedBindingProjections = (alphabet.bindingProjections ?? []).map(plan => JSON.stringify(plan))
|
|
3856
|
+
const expectedTerminalProjections = actual.terminalProjections.map(plan => JSON.stringify(plan))
|
|
3857
|
+
const suppliedTerminalProjections = (alphabet.terminalProjections ?? []).map(plan => JSON.stringify(plan))
|
|
3858
|
+
const familySignature = (family: LexicalDecisionFamily): string =>
|
|
3859
|
+
`${family.id}\u0000${family.semanticKey}\u0000${JSON.stringify(family.ir)}`
|
|
3860
|
+
const expectedFamilies = actual.decisionFamilies.map(familySignature)
|
|
3861
|
+
const suppliedFamilies = alphabet.decisionFamilies.map(familySignature)
|
|
3862
|
+
const outcomeSignature = (outcome: LexicalDecisionOutcome): string =>
|
|
3863
|
+
`${outcome.id}\u0000${outcome.familyId}\u0000${outcome.semanticKey}\u0000${JSON.stringify(outcome.view)}`
|
|
3864
|
+
const expectedOutcomes = actual.decisionOutcomes.map(outcomeSignature)
|
|
3865
|
+
const suppliedOutcomes = alphabet.decisionOutcomes.map(outcomeSignature)
|
|
3866
|
+
const decisionSignature = (decision: LexicalDecisionSite): string => JSON.stringify(decision)
|
|
3867
|
+
const expectedDecisions = actual.decisions.map(decisionSignature)
|
|
3868
|
+
const suppliedDecisions = alphabet.decisions.map(decisionSignature)
|
|
3869
|
+
const c2Signature = (value: unknown): string => JSON.stringify(value, (_key, entry: unknown) =>
|
|
3870
|
+
typeof entry === 'function' ? String(entry) : entry)
|
|
3871
|
+
const expectedC2 = c2Signature(actual.decisionEffectPlan)
|
|
3872
|
+
const suppliedC2 = c2Signature(alphabet.decisionEffectPlan ?? {})
|
|
3873
|
+
const planSignature = (plan: LexicalTransitionDiagnosticPlan): string => JSON.stringify(plan)
|
|
3874
|
+
const expectedPlans = actual.transitionDiagnostics.map(planSignature)
|
|
3875
|
+
const suppliedPlans = (alphabet.transitionDiagnostics ?? []).map(planSignature)
|
|
3876
|
+
const expectedControlPlans = actual.controlPlans.map(plan => JSON.stringify(plan))
|
|
3877
|
+
const suppliedControlPlans = (alphabet.controlPlans ?? []).map(plan => JSON.stringify(plan))
|
|
3878
|
+
const expectedBoundaryPlans = actual.boundaryPlans.map(plan => JSON.stringify(plan))
|
|
3879
|
+
const suppliedBoundaryPlans = (alphabet.boundaryPlans ?? []).map(plan => JSON.stringify(plan))
|
|
3880
|
+
const expectedMaterializationPlans = actual.materializationPlans.map(plan => JSON.stringify(plan))
|
|
3881
|
+
const suppliedMaterializationPlans = (alphabet.materializationPlans ?? []).map(plan => JSON.stringify(plan))
|
|
3882
|
+
const expectedGrammarSpecs = actual.grammarWrapperSpecs.map(plan => JSON.stringify(plan))
|
|
3883
|
+
const suppliedGrammarSpecs = (alphabet.grammarWrapperSpecs ?? []).map(plan => JSON.stringify(plan))
|
|
3884
|
+
const expectedCaptureKinds = actual.grammarCaptureTriviaKinds.map(plan => JSON.stringify(plan))
|
|
3885
|
+
const suppliedCaptureKinds = (alphabet.grammarCaptureTriviaKinds ?? []).map(plan => JSON.stringify(plan))
|
|
3886
|
+
const expectedTopologies = actual.boundaryTopologies.map(plan => JSON.stringify(plan))
|
|
3887
|
+
const suppliedTopologies = (alphabet.boundaryTopologies ?? []).map(plan => JSON.stringify(plan))
|
|
3888
|
+
const expectedContexts = actual.contextSnapshots.map(plan => JSON.stringify(plan))
|
|
3889
|
+
const suppliedContexts = (alphabet.contextSnapshots ?? []).map(plan => JSON.stringify(plan))
|
|
3890
|
+
const expectedOwnSkip = actual.ownSkipPlans.map(plan => JSON.stringify(plan))
|
|
3891
|
+
const suppliedOwnSkip = (alphabet.ownSkipPlans ?? []).map(plan => JSON.stringify(plan))
|
|
3892
|
+
const expectedScanPolicies = actual.scanConsumerPolicies.map(plan => JSON.stringify(plan))
|
|
3893
|
+
const suppliedScanPolicies = (alphabet.scanConsumerPolicies ?? []).map(plan => JSON.stringify(plan))
|
|
3894
|
+
if (expectedKeys.length !== suppliedKeys.length
|
|
3895
|
+
|| expectedKeys.some((key, index) => key !== suppliedKeys[index])
|
|
3896
|
+
|| expectedLanguages.length !== suppliedLanguages.length
|
|
3897
|
+
|| expectedLanguages.some((key, index) => key !== suppliedLanguages[index])
|
|
3898
|
+
|| expectedEdges.length !== suppliedEdges.length
|
|
3899
|
+
|| expectedEdges.some((key, index) => key !== suppliedEdges[index])
|
|
3900
|
+
|| expectedBindingProjections.length !== suppliedBindingProjections.length
|
|
3901
|
+
|| expectedBindingProjections.some((key, index) => key !== suppliedBindingProjections[index])
|
|
3902
|
+
|| expectedTerminalProjections.length !== suppliedTerminalProjections.length
|
|
3903
|
+
|| expectedTerminalProjections.some((key, index) => key !== suppliedTerminalProjections[index])
|
|
3904
|
+
|| expectedFamilies.length !== suppliedFamilies.length
|
|
3905
|
+
|| expectedFamilies.some((key, index) => key !== suppliedFamilies[index])
|
|
3906
|
+
|| expectedOutcomes.length !== suppliedOutcomes.length
|
|
3907
|
+
|| expectedOutcomes.some((key, index) => key !== suppliedOutcomes[index])
|
|
3908
|
+
|| expectedDecisions.length !== suppliedDecisions.length
|
|
3909
|
+
|| expectedDecisions.some((key, index) => key !== suppliedDecisions[index])
|
|
3910
|
+
|| expectedC2 !== suppliedC2
|
|
3911
|
+
|| actual.decisionEffectPlan.decisionCallbackAuthorities.length
|
|
3912
|
+
!== (alphabet.decisionEffectPlan?.decisionCallbackAuthorities.length ?? 0)
|
|
3913
|
+
|| actual.decisionEffectPlan.decisionCallbackAuthorities.some((entry, index) =>
|
|
3914
|
+
entry.callback !== alphabet.decisionEffectPlan?.decisionCallbackAuthorities[index]?.callback)
|
|
3915
|
+
|| expectedPlans.length !== suppliedPlans.length
|
|
3916
|
+
|| expectedPlans.some((key, index) => key !== suppliedPlans[index])
|
|
3917
|
+
|| expectedControlPlans.length !== suppliedControlPlans.length
|
|
3918
|
+
|| expectedControlPlans.some((key, index) => key !== suppliedControlPlans[index])
|
|
3919
|
+
|| expectedBoundaryPlans.length !== suppliedBoundaryPlans.length
|
|
3920
|
+
|| expectedBoundaryPlans.some((key, index) => key !== suppliedBoundaryPlans[index])
|
|
3921
|
+
|| expectedMaterializationPlans.length !== suppliedMaterializationPlans.length
|
|
3922
|
+
|| expectedMaterializationPlans.some((key, index) => key !== suppliedMaterializationPlans[index])
|
|
3923
|
+
|| expectedGrammarSpecs.length !== suppliedGrammarSpecs.length
|
|
3924
|
+
|| expectedGrammarSpecs.some((key, index) => key !== suppliedGrammarSpecs[index])
|
|
3925
|
+
|| expectedCaptureKinds.length !== suppliedCaptureKinds.length
|
|
3926
|
+
|| expectedCaptureKinds.some((key, index) => key !== suppliedCaptureKinds[index])
|
|
3927
|
+
|| expectedTopologies.length !== suppliedTopologies.length
|
|
3928
|
+
|| expectedTopologies.some((key, index) => key !== suppliedTopologies[index])
|
|
3929
|
+
|| expectedContexts.length !== suppliedContexts.length
|
|
3930
|
+
|| expectedContexts.some((key, index) => key !== suppliedContexts[index])
|
|
3931
|
+
|| expectedOwnSkip.length !== suppliedOwnSkip.length
|
|
3932
|
+
|| expectedOwnSkip.some((key, index) => key !== suppliedOwnSkip[index])
|
|
3933
|
+
|| expectedScanPolicies.length !== suppliedScanPolicies.length
|
|
3934
|
+
|| expectedScanPolicies.some((key, index) => key !== suppliedScanPolicies[index])) {
|
|
3935
|
+
throw new Error('parseman: lexical capability census is incomplete after final grammar resolution')
|
|
3936
|
+
}
|
|
3937
|
+
}
|
|
3938
|
+
|
|
3939
|
+
/** Phase-A only: inventory obligations without constructing unused runtime families/sites. */
|
|
3940
|
+
export function collectLexicalCapabilities(
|
|
3941
|
+
roots: ReadonlyArray<Combinator<unknown>>,
|
|
3942
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
3943
|
+
): LexicalCapabilityInventory {
|
|
3944
|
+
const inventory = lexicalCapabilityInventory(roots, resolve)
|
|
3945
|
+
const { languages: capabilityLanguages, ...fields } = inventory
|
|
3946
|
+
return {
|
|
3947
|
+
...fields,
|
|
3948
|
+
capabilityLanguages,
|
|
3949
|
+
capabilityComplete: inventory.capabilities.every(site => site.status.kind !== 'gap')
|
|
3950
|
+
&& inventory.bindingEdges.every(edge => edge.status.kind !== 'gap')
|
|
3951
|
+
&& inventory.decisionEffectPlan.decisionEffects.every(effect =>
|
|
3952
|
+
effect.phase.executableLowering.kind !== 'gap')
|
|
3953
|
+
&& inventory.scanConsumerPolicies.every(policy => policy.status.kind !== 'gap'),
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3956
|
+
|
|
3957
|
+
function tokenBoundary(
|
|
3958
|
+
parser: Combinator<unknown>,
|
|
3959
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
3960
|
+
seen: Set<Combinator<unknown>> = new Set(),
|
|
3961
|
+
selectorEffects = false,
|
|
3962
|
+
): { token: Combinator<unknown>; selectorEffects: boolean } | undefined {
|
|
3963
|
+
if (seen.has(parser)) return undefined
|
|
3964
|
+
seen.add(parser)
|
|
3965
|
+
const def = parser._def
|
|
3966
|
+
if (def.tag === 'token') return { token: parser, selectorEffects }
|
|
3967
|
+
if (def.tag === 'lazy') {
|
|
3968
|
+
const target = resolvedLazyTarget(parser, resolve)
|
|
3969
|
+
return target === undefined ? undefined : tokenBoundary(target, resolve, seen, true)
|
|
3970
|
+
}
|
|
3971
|
+
// Value-preserving does NOT mean effect-free: label/field/expect/attempt and
|
|
3972
|
+
// scopes carry diagnostics, capture, commitment, or state. Do not inspect
|
|
3973
|
+
// through them until a consumer proves those effects still execute.
|
|
3974
|
+
return undefined
|
|
3975
|
+
}
|
|
3976
|
+
|
|
3977
|
+
/**
|
|
3978
|
+
* Catalog authored lexical tokens and their dispatch views without confusing
|
|
3979
|
+
* private child terminals for source tokens. This is metadata only: consumers
|
|
3980
|
+
* still have to lower the canonical IR in every shipping engine before enabling
|
|
3981
|
+
* an admission path.
|
|
3982
|
+
*/
|
|
3983
|
+
export function collectLexicalAlphabet(
|
|
3984
|
+
roots: ReadonlyArray<Combinator<unknown>>,
|
|
3985
|
+
resolve?: (name: string) => Combinator<unknown> | undefined,
|
|
3986
|
+
): LexicalAlphabet {
|
|
3987
|
+
const capabilityInventory = collectLexicalCapabilities(roots, resolve)
|
|
3988
|
+
const { capabilities } = capabilityInventory
|
|
3989
|
+
const capabilityFamilyIdByKey = new Map(capabilityInventory.decisionFamilies.map(family =>
|
|
3990
|
+
[family.semanticKey, family.id]))
|
|
3991
|
+
const capabilityTokenFamilies = new Map<Combinator<unknown>, Map<number, LexicalDecisionFamily>>()
|
|
3992
|
+
for (const site of capabilities) {
|
|
3993
|
+
if (site.atom !== 'token' || site.obligations.recognition.representation.kind !== 'complete') continue
|
|
3994
|
+
const semanticKey = site.semanticKey.slice(2)
|
|
3995
|
+
const familyId = capabilityFamilyIdByKey.get(semanticKey)
|
|
3996
|
+
const family = familyId === undefined ? undefined
|
|
3997
|
+
: capabilityInventory.decisionFamilies.find(entry => entry.id === familyId)
|
|
3998
|
+
if (family === undefined) {
|
|
3999
|
+
throw new Error('parseman: completed token occurrence has no canonical capability family')
|
|
4000
|
+
}
|
|
4001
|
+
let byId = capabilityTokenFamilies.get(site.parser)
|
|
4002
|
+
if (byId === undefined) capabilityTokenFamilies.set(site.parser, byId = new Map())
|
|
4003
|
+
byId.set(family.id, family)
|
|
4004
|
+
}
|
|
4005
|
+
const tokenParsers: Combinator<unknown>[] = []
|
|
4006
|
+
const dispatchParsers: Combinator<unknown>[] = []
|
|
4007
|
+
const seenTop = new Set<Combinator<unknown>>()
|
|
4008
|
+
const seenOwned = new Set<Combinator<unknown>>()
|
|
4009
|
+
|
|
4010
|
+
const walk = (parser: Combinator<unknown>, owner?: Combinator<unknown>): void => {
|
|
4011
|
+
const seen = owner === undefined ? seenTop : seenOwned
|
|
4012
|
+
if (seen.has(parser)) return
|
|
4013
|
+
seen.add(parser)
|
|
4014
|
+
if (owner === undefined && parser._def.tag === 'token') tokenParsers.push(parser)
|
|
4015
|
+
if (owner === undefined && parser._def.tag === 'dispatch') dispatchParsers.push(parser)
|
|
4016
|
+
const nextOwner = owner ?? (parser._def.tag === 'token' ? parser : undefined)
|
|
4017
|
+
for (const entry of tokenChildren(parser, resolve)) walk(entry, nextOwner)
|
|
4018
|
+
}
|
|
4019
|
+
for (const root of roots) walk(root)
|
|
4020
|
+
|
|
4021
|
+
const familyIdOf = new Map<Combinator<unknown>, number>()
|
|
4022
|
+
const recognizers: LexicalRecognizer[] = []
|
|
4023
|
+
const diagnostics: LexicalDiagnosticPlan[] = []
|
|
4024
|
+
const recognizerByKey = new Map<string, number>()
|
|
4025
|
+
const sites: LexicalTokenSite[] = []
|
|
4026
|
+
for (const parser of tokenParsers) {
|
|
4027
|
+
const def = parser._def
|
|
4028
|
+
if (def.tag !== 'token') continue
|
|
4029
|
+
const candidateFamilies = capabilityTokenFamilies.get(parser)
|
|
4030
|
+
if (candidateFamilies === undefined) {
|
|
4031
|
+
const refused = normalizeBalancedLexical(parser, resolve)
|
|
4032
|
+
?? normalizeLexical(def.parser, resolve, new Set())
|
|
4033
|
+
if ('refusal' in refused) {
|
|
4034
|
+
sites.push({ parser, body: def.parser, refusal: refused.refusal })
|
|
4035
|
+
continue
|
|
4036
|
+
}
|
|
4037
|
+
if (matchesEmpty(def.parser)) {
|
|
4038
|
+
sites.push({ parser, body: def.parser, refusal: 'token body may match empty' })
|
|
4039
|
+
continue
|
|
4040
|
+
}
|
|
4041
|
+
throw new Error('parseman: successful legacy recognizer has no canonical capability family')
|
|
4042
|
+
}
|
|
4043
|
+
if (candidateFamilies.size !== 1) {
|
|
4044
|
+
sites.push({
|
|
4045
|
+
parser, body: def.parser,
|
|
4046
|
+
refusal: 'token parser is reused in incompatible lexical contexts',
|
|
4047
|
+
})
|
|
4048
|
+
continue
|
|
4049
|
+
}
|
|
4050
|
+
if (matchesEmpty(def.parser)) {
|
|
4051
|
+
sites.push({ parser, body: def.parser, refusal: 'token body may match empty' })
|
|
4052
|
+
continue
|
|
4053
|
+
}
|
|
4054
|
+
const diagnosticId = diagnostics.length
|
|
4055
|
+
diagnostics.push({ id: diagnosticId, body: def.parser })
|
|
4056
|
+
const canonicalFamily = [...candidateFamilies.values()][0]!
|
|
4057
|
+
const ir = canonicalFamily.ir
|
|
4058
|
+
const key = JSON.stringify(ir)
|
|
4059
|
+
let recognizerId = recognizerByKey.get(key)
|
|
4060
|
+
if (recognizerId === undefined) {
|
|
4061
|
+
recognizerId = recognizers.length
|
|
4062
|
+
recognizerByKey.set(key, recognizerId)
|
|
4063
|
+
recognizers.push({
|
|
4064
|
+
id: recognizerId, key, ir, capabilityFamilyId: canonicalFamily.id,
|
|
4065
|
+
})
|
|
4066
|
+
}
|
|
4067
|
+
const familyId = FIRST_LEXICAL_FAMILY_ID + recognizerId
|
|
4068
|
+
familyIdOf.set(parser, familyId)
|
|
4069
|
+
sites.push({ parser, body: def.parser, diagnosticId, recognizerId, familyId })
|
|
4070
|
+
}
|
|
4071
|
+
const families: LexicalTokenFamily[] = recognizers.map(recognizer => ({
|
|
4072
|
+
id: FIRST_LEXICAL_FAMILY_ID + recognizer.id,
|
|
4073
|
+
recognizerId: recognizer.id,
|
|
4074
|
+
}))
|
|
4075
|
+
|
|
4076
|
+
let nextOutcomeId = FIRST_LEXICAL_FAMILY_ID + families.length
|
|
4077
|
+
const outcomes: LexicalOutcomeSpec[] = []
|
|
4078
|
+
const outcomeIdByKey = new Map<string, number>()
|
|
4079
|
+
const internOutcome = (familyId: number, match: LexicalOutcomeMatch): number => {
|
|
4080
|
+
if (match.kind === 'exact' && match.values.length !== 1) {
|
|
4081
|
+
throw new Error('parseman: global exact lexical outcomes must be atomic')
|
|
4082
|
+
}
|
|
4083
|
+
const key = `${familyId}\u0000${canonicalLexicalOutcomeKey(match)}`
|
|
4084
|
+
const prior = outcomeIdByKey.get(key)
|
|
4085
|
+
if (prior !== undefined) return prior
|
|
4086
|
+
const id = nextOutcomeId++
|
|
4087
|
+
outcomeIdByKey.set(key, id)
|
|
4088
|
+
outcomes.push({ id, familyId, match })
|
|
4089
|
+
return id
|
|
4090
|
+
}
|
|
4091
|
+
const classifiers: LexicalTokenClassifier[] = []
|
|
4092
|
+
for (const parser of dispatchParsers) {
|
|
4093
|
+
const def = parser._def
|
|
4094
|
+
if (def.tag !== 'dispatch') continue
|
|
4095
|
+
const boundary = tokenBoundary(def.selector, resolve)
|
|
4096
|
+
const familyId = boundary === undefined ? undefined : familyIdOf.get(boundary.token)
|
|
4097
|
+
if (familyId === undefined) continue
|
|
4098
|
+
const outcomes: LexicalTokenOutcome[] = []
|
|
4099
|
+
const routes: LexicalTokenRoute[] = []
|
|
4100
|
+
const exclusions: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>[] = []
|
|
4101
|
+
for (const entry of def.cases) {
|
|
4102
|
+
// Exact subtype ids are ATOMIC even when one dispatch arm groups keys.
|
|
4103
|
+
// The site route is represented by multiple outcomes pointing at the same
|
|
4104
|
+
// parser, so URL_OPEN keeps one global id whether another site groups it
|
|
4105
|
+
// with VAR_OPEN/CALC_OPEN or routes it alone.
|
|
4106
|
+
const matches: LexicalOutcomeMatch[] = []
|
|
4107
|
+
const acceptedIds: number[] = []
|
|
4108
|
+
for (const key of entry.keys) {
|
|
4109
|
+
const match: LexicalOutcomeMatch = {
|
|
4110
|
+
kind: 'exact', values: [key], caseInsensitive: entry.caseInsensitive,
|
|
4111
|
+
}
|
|
4112
|
+
const id = internOutcome(familyId, match)
|
|
4113
|
+
exclusions.push(match)
|
|
4114
|
+
matches.push(match)
|
|
4115
|
+
acceptedIds.push(id)
|
|
4116
|
+
outcomes.push({ id, match })
|
|
4117
|
+
}
|
|
4118
|
+
routes.push({
|
|
4119
|
+
index: routes.length, kind: 'exact', acceptedIds, matches,
|
|
4120
|
+
parser: entry.parser, usesRouted: branchUsesRouted(entry),
|
|
4121
|
+
})
|
|
4122
|
+
}
|
|
4123
|
+
for (const entry of def.matchers ?? []) {
|
|
4124
|
+
const match: LexicalOutcomeMatch = entry.kind === 'matches'
|
|
4125
|
+
? { kind: 'matches', value: entry.value, flags: entry.flags ?? '', caseInsensitive: entry.caseInsensitive }
|
|
4126
|
+
: { kind: entry.kind, value: entry.value, caseInsensitive: entry.caseInsensitive }
|
|
4127
|
+
exclusions.push(match)
|
|
4128
|
+
const id = internOutcome(familyId, match)
|
|
4129
|
+
outcomes.push({ id, match })
|
|
4130
|
+
routes.push({
|
|
4131
|
+
index: routes.length, kind: 'matcher', acceptedIds: [id], matches: [match],
|
|
4132
|
+
parser: entry.parser, usesRouted: branchUsesRouted(entry),
|
|
4133
|
+
})
|
|
4134
|
+
}
|
|
4135
|
+
if (def.otherwise !== undefined) {
|
|
4136
|
+
const match: LexicalOutcomeMatch = { kind: 'otherwise', excluding: exclusions }
|
|
4137
|
+
const id = internOutcome(familyId, match)
|
|
4138
|
+
outcomes.push({ id, match })
|
|
4139
|
+
routes.push({
|
|
4140
|
+
index: routes.length, kind: 'otherwise', acceptedIds: [id], matches: [match],
|
|
4141
|
+
parser: def.otherwise,
|
|
4142
|
+
usesRouted: branchUsesRouted({ parser: def.otherwise, usesRouted: def.otherwiseUsesRouted }),
|
|
4143
|
+
})
|
|
4144
|
+
}
|
|
4145
|
+
for (const route of routes) {
|
|
4146
|
+
if (route.matches.length !== route.acceptedIds.length) {
|
|
4147
|
+
throw new Error('parseman: lexical route match/id arity mismatch')
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
classifiers.push({
|
|
4151
|
+
dispatch: parser, familyId, selectorEffects: boundary?.selectorEffects ?? false, outcomes, routes,
|
|
4152
|
+
})
|
|
4153
|
+
}
|
|
4154
|
+
|
|
4155
|
+
/*
|
|
4156
|
+
* IDs ARE CONTENT-ORDERED, not discovery-ordered. The collector walks the
|
|
4157
|
+
* final winner graph once, but compose(), a rule-map literal, and the macro
|
|
4158
|
+
* evaluator are allowed to present equal roots in different insertion order.
|
|
4159
|
+
* Sorting the already-canonical recognizer keys and family-qualified outcome
|
|
4160
|
+
* keys makes those routes agree without publishing a hash or an authored
|
|
4161
|
+
* token-instance identity in the artifact.
|
|
4162
|
+
*/
|
|
4163
|
+
const recognizerOrder = [...recognizers].sort((a, b) =>
|
|
4164
|
+
a.key < b.key ? -1 : a.key > b.key ? 1 : 0)
|
|
4165
|
+
const recognizerId = new Map(recognizerOrder.map((entry, index) => [entry.id, index]))
|
|
4166
|
+
const stableRecognizers = recognizerOrder.map((entry, id) => ({ ...entry, id }))
|
|
4167
|
+
const stableFamilies = stableRecognizers.map(entry => ({
|
|
4168
|
+
id: FIRST_LEXICAL_FAMILY_ID + entry.id,
|
|
4169
|
+
recognizerId: entry.id,
|
|
4170
|
+
}))
|
|
4171
|
+
const stableFamilyId = (oldFamilyId: number): number => {
|
|
4172
|
+
const oldRecognizerId = oldFamilyId - FIRST_LEXICAL_FAMILY_ID
|
|
4173
|
+
const id = recognizerId.get(oldRecognizerId)
|
|
4174
|
+
if (id === undefined) throw new Error('parseman: lexical family has no recognizer')
|
|
4175
|
+
return FIRST_LEXICAL_FAMILY_ID + id
|
|
4176
|
+
}
|
|
4177
|
+
const stableFamilyIdOf = new Map<Combinator<unknown>, number>()
|
|
4178
|
+
for (const [parser, familyId] of familyIdOf) stableFamilyIdOf.set(parser, stableFamilyId(familyId))
|
|
4179
|
+
const stableSites = sites.map(site => site.familyId === undefined ? site : ({
|
|
4180
|
+
...site,
|
|
4181
|
+
familyId: stableFamilyId(site.familyId),
|
|
4182
|
+
recognizerId: recognizerId.get(site.recognizerId!)!,
|
|
4183
|
+
}))
|
|
4184
|
+
|
|
4185
|
+
const orderedOutcomes = [...outcomes].sort((a, b) => {
|
|
4186
|
+
const ak = `${stableFamilyId(a.familyId)}\u0000${canonicalLexicalOutcomeKey(a.match)}`
|
|
4187
|
+
const bk = `${stableFamilyId(b.familyId)}\u0000${canonicalLexicalOutcomeKey(b.match)}`
|
|
4188
|
+
return ak < bk ? -1 : ak > bk ? 1 : 0
|
|
4189
|
+
})
|
|
4190
|
+
const oldOutcomeToStable = new Map<number, number>()
|
|
4191
|
+
const firstOutcomeId = FIRST_LEXICAL_FAMILY_ID + stableFamilies.length
|
|
4192
|
+
const stableOutcomes = orderedOutcomes.map((entry, index) => {
|
|
4193
|
+
const id = firstOutcomeId + index
|
|
4194
|
+
oldOutcomeToStable.set(entry.id, id)
|
|
4195
|
+
return { ...entry, id, familyId: stableFamilyId(entry.familyId) }
|
|
4196
|
+
})
|
|
4197
|
+
const stableClassifiers = classifiers.map(classifier => ({
|
|
4198
|
+
...classifier,
|
|
4199
|
+
familyId: stableFamilyId(classifier.familyId),
|
|
4200
|
+
outcomes: classifier.outcomes.map(outcome => ({
|
|
4201
|
+
...outcome,
|
|
4202
|
+
id: oldOutcomeToStable.get(outcome.id)!,
|
|
4203
|
+
})),
|
|
4204
|
+
routes: classifier.routes.map(route => ({
|
|
4205
|
+
...route,
|
|
4206
|
+
acceptedIds: route.acceptedIds.map(id => oldOutcomeToStable.get(id)!),
|
|
4207
|
+
})),
|
|
4208
|
+
}))
|
|
4209
|
+
|
|
4210
|
+
// Deliberately do not return `primitiveKernels`: collectAlphabet() remains a
|
|
4211
|
+
// separate historical/kernel API, while this is the only family/site seam.
|
|
4212
|
+
// A choice-admission consumer therefore cannot accidentally publish a child
|
|
4213
|
+
// literal/regex id by reading a field from LexicalAlphabet.
|
|
4214
|
+
return {
|
|
4215
|
+
...capabilityInventory,
|
|
4216
|
+
recognizers: stableRecognizers,
|
|
4217
|
+
diagnostics,
|
|
4218
|
+
families: stableFamilies,
|
|
4219
|
+
sites: stableSites,
|
|
4220
|
+
outcomes: stableOutcomes,
|
|
4221
|
+
classifiers: stableClassifiers,
|
|
4222
|
+
familyIdOf: stableFamilyIdOf,
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
4225
|
+
|
|
4226
|
+
function serializeLexicalIr(ir: LexicalIr, out: number[], constant: (value: unknown) => number): void {
|
|
4227
|
+
const start = out.length
|
|
4228
|
+
const begin = (kind: number): void => { out.push(kind, 0) }
|
|
4229
|
+
switch (ir.kind) {
|
|
4230
|
+
case 'literal':
|
|
4231
|
+
begin(0); out.push(constant(ir.value), ir.caseInsensitive ? 1 : 0)
|
|
4232
|
+
break
|
|
4233
|
+
case 'keywords':
|
|
4234
|
+
begin(1)
|
|
4235
|
+
out.push(ir.caseInsensitive ? 1 : 0, ir.boundary === undefined ? -1 : constant(ir.boundary), ir.words.length)
|
|
4236
|
+
for (const word of ir.words) out.push(constant(word))
|
|
4237
|
+
break
|
|
4238
|
+
case 'regex':
|
|
4239
|
+
begin(2); out.push(constant(new RegExp(ir.source, ir.flags)))
|
|
4240
|
+
break
|
|
4241
|
+
case 'sequence':
|
|
4242
|
+
begin(3); out.push(ir.parts.length)
|
|
4243
|
+
for (const part of ir.parts) serializeLexicalIr(part, out, constant)
|
|
4244
|
+
break
|
|
4245
|
+
case 'choice':
|
|
4246
|
+
begin(4); out.push(ir.arms.length)
|
|
4247
|
+
for (const arm of ir.arms) serializeLexicalIr(arm, out, constant)
|
|
4248
|
+
break
|
|
4249
|
+
case 'repeat':
|
|
4250
|
+
begin(5)
|
|
4251
|
+
out.push(ir.min, ir.max ?? -1, ir.greedy ? 1 : 0, ir.mode === 'possessive' ? 0 : 1)
|
|
4252
|
+
serializeLexicalIr(ir.body, out, constant)
|
|
4253
|
+
break
|
|
4254
|
+
case 'assert':
|
|
4255
|
+
begin(6); out.push(ir.positive ? 1 : 0)
|
|
4256
|
+
serializeLexicalIr(ir.body, out, constant)
|
|
4257
|
+
break
|
|
4258
|
+
case 'balanced':
|
|
4259
|
+
begin(7)
|
|
4260
|
+
out.push(constant(ir.open), constant(ir.close), ir.strict ? 1 : 0, ir.raw ? 1 : 0, ir.skip.length)
|
|
4261
|
+
for (const entry of ir.skip) serializeLexicalIr(entry, out, constant)
|
|
4262
|
+
break
|
|
4263
|
+
}
|
|
4264
|
+
out[start + 1] = out.length - start
|
|
4265
|
+
}
|
|
4266
|
+
|
|
4267
|
+
/**
|
|
4268
|
+
* Project the compiler graph into compact numeric pools. The caller supplies
|
|
4269
|
+
* only already-relocated table-site numbers and its existing const-pool intern;
|
|
4270
|
+
* no combinator or identity map crosses this boundary.
|
|
4271
|
+
*/
|
|
4272
|
+
export function serializeLexicalPlan(
|
|
4273
|
+
alphabet: LexicalAlphabet,
|
|
4274
|
+
constant: (value: unknown) => number,
|
|
4275
|
+
tokenSites: readonly number[],
|
|
4276
|
+
dispatchSites: ReadonlyArray<{ readonly dsp: number; readonly classifier: LexicalTokenClassifier }>,
|
|
4277
|
+
): NumericLexicalPlan | undefined {
|
|
4278
|
+
// Phase B is globally disabled until phase A has a total replacement body
|
|
4279
|
+
// for every reachable token-capable atom in every supported variant. A
|
|
4280
|
+
// partially supported graph is never serialized as a cheaper token plan.
|
|
4281
|
+
if (!alphabet.capabilityComplete) return undefined
|
|
4282
|
+
if (tokenSites.length === 0) return undefined
|
|
4283
|
+
const recognizerOffsets: number[] = []
|
|
4284
|
+
const recognizerData: number[] = []
|
|
4285
|
+
for (const recognizer of alphabet.recognizers) {
|
|
4286
|
+
if (recognizer.id !== recognizerOffsets.length) throw new Error('parseman: lexical recognizer ids are not dense')
|
|
4287
|
+
recognizerOffsets.push(recognizerData.length)
|
|
4288
|
+
serializeLexicalIr(recognizer.ir, recognizerData, constant)
|
|
4289
|
+
}
|
|
4290
|
+
|
|
4291
|
+
const outcomeOffsets: number[] = []
|
|
4292
|
+
const outcomeData: number[] = []
|
|
4293
|
+
for (const outcome of alphabet.outcomes) {
|
|
4294
|
+
outcomeOffsets.push(outcomeData.length)
|
|
4295
|
+
const match = outcome.match
|
|
4296
|
+
switch (match.kind) {
|
|
4297
|
+
case 'exact':
|
|
4298
|
+
if (match.values.length !== 1) throw new Error('parseman: lexical exact outcome is not atomic')
|
|
4299
|
+
outcomeData.push(outcome.id, outcome.familyId, 0, constant(match.values[0]!), match.caseInsensitive ? 1 : 0)
|
|
4300
|
+
break
|
|
4301
|
+
case 'startsWith':
|
|
4302
|
+
outcomeData.push(outcome.id, outcome.familyId, 1, constant(match.value), match.caseInsensitive ? 1 : 0)
|
|
4303
|
+
break
|
|
4304
|
+
case 'endsWith':
|
|
4305
|
+
outcomeData.push(outcome.id, outcome.familyId, 2, constant(match.value), match.caseInsensitive ? 1 : 0)
|
|
4306
|
+
break
|
|
4307
|
+
case 'matches': {
|
|
4308
|
+
const rawFlags = match.caseInsensitive && !match.flags.includes('i') ? `${match.flags}i` : match.flags
|
|
4309
|
+
const flags = new RegExp('', rawFlags.replace(/g/g, '')).flags
|
|
4310
|
+
outcomeData.push(outcome.id, outcome.familyId, 3, constant(new RegExp(match.value, flags)))
|
|
4311
|
+
break
|
|
4312
|
+
}
|
|
4313
|
+
case 'otherwise':
|
|
4314
|
+
outcomeData.push(outcome.id, outcome.familyId, 4)
|
|
4315
|
+
break
|
|
4316
|
+
}
|
|
4317
|
+
}
|
|
4318
|
+
|
|
4319
|
+
const sites: number[] = []
|
|
4320
|
+
const routes: number[] = []
|
|
4321
|
+
const accepted: number[] = []
|
|
4322
|
+
for (const { dsp, classifier } of [...dispatchSites].sort((a, b) => a.dsp - b.dsp)) {
|
|
4323
|
+
const routeOffset = routes.length
|
|
4324
|
+
for (const route of classifier.routes) {
|
|
4325
|
+
const acceptedOffset = accepted.length
|
|
4326
|
+
accepted.push(...route.acceptedIds)
|
|
4327
|
+
const kind = route.kind === 'exact' ? 0 : route.kind === 'matcher' ? 1 : 2
|
|
4328
|
+
routes.push(route.index, kind | (route.usesRouted ? 4 : 0), acceptedOffset, route.acceptedIds.length)
|
|
4329
|
+
}
|
|
4330
|
+
sites.push(dsp, classifier.familyId, routeOffset, classifier.routes.length)
|
|
4331
|
+
}
|
|
4332
|
+
for (let i = 0; i < tokenSites.length; i += 2) {
|
|
4333
|
+
if (tokenSites[i + 1]! < FIRST_LEXICAL_FAMILY_ID) {
|
|
4334
|
+
throw new Error('parseman: lexical token site used a primitive-terminal id')
|
|
4335
|
+
}
|
|
4336
|
+
}
|
|
4337
|
+
return {
|
|
4338
|
+
recognizerOffsets, recognizerData, outcomeOffsets, outcomeData,
|
|
4339
|
+
tokenSites: [...tokenSites], sites, routes, accepted,
|
|
4340
|
+
}
|
|
4341
|
+
}
|
|
4342
|
+
|
|
4343
|
+
function foldAsciiCode(code: number): number {
|
|
4344
|
+
return code >= 65 && code <= 90 ? code + 32 : code
|
|
4345
|
+
}
|
|
4346
|
+
|
|
4347
|
+
function rangeEquals(input: string, start: number, end: number, value: string, folded: boolean): boolean {
|
|
4348
|
+
if (end - start !== value.length) return false
|
|
4349
|
+
for (let i = 0; i < value.length; i++) {
|
|
4350
|
+
const actual = input.charCodeAt(start + i)
|
|
4351
|
+
const expected = value.charCodeAt(i)
|
|
4352
|
+
if ((folded ? foldAsciiCode(actual) : actual) !== (folded ? foldAsciiCode(expected) : expected)) return false
|
|
4353
|
+
}
|
|
4354
|
+
return true
|
|
4355
|
+
}
|
|
4356
|
+
|
|
4357
|
+
function rangeStartsWith(input: string, start: number, end: number, value: string, folded: boolean): boolean {
|
|
4358
|
+
return start + value.length <= end && rangeEquals(input, start, start + value.length, value, folded)
|
|
4359
|
+
}
|
|
4360
|
+
|
|
4361
|
+
function rangeEndsWith(input: string, start: number, end: number, value: string, folded: boolean): boolean {
|
|
4362
|
+
return end - value.length >= start && rangeEquals(input, end - value.length, end, value, folded)
|
|
4363
|
+
}
|
|
4364
|
+
|
|
4365
|
+
function lexicalOutcomeMatches(
|
|
4366
|
+
match: Exclude<LexicalOutcomeMatch, { kind: 'otherwise' }>,
|
|
4367
|
+
input: string,
|
|
4368
|
+
start: number,
|
|
4369
|
+
end: number,
|
|
4370
|
+
): boolean {
|
|
4371
|
+
if (match.kind === 'exact') {
|
|
4372
|
+
return match.values.some(value => rangeEquals(input, start, end, value, match.caseInsensitive))
|
|
4373
|
+
}
|
|
4374
|
+
if (match.kind === 'startsWith') return rangeStartsWith(input, start, end, match.value, match.caseInsensitive)
|
|
4375
|
+
if (match.kind === 'endsWith') return rangeEndsWith(input, start, end, match.value, match.caseInsensitive)
|
|
4376
|
+
if ('flags' in match) {
|
|
4377
|
+
const flags = match.caseInsensitive && !match.flags.includes('i') ? `${match.flags}i` : match.flags
|
|
4378
|
+
return new RegExp(match.value, flags).test(input.slice(start, end))
|
|
4379
|
+
}
|
|
4380
|
+
return false
|
|
4381
|
+
}
|
|
4382
|
+
|
|
4383
|
+
/**
|
|
4384
|
+
* Metadata/reference oracle for compatible same-range views. Exact/prefix/suffix
|
|
4385
|
+
* checks read char codes directly; regex matcher views slice only in this cold
|
|
4386
|
+
* oracle. Production token cursors must lower matcher plans without treating
|
|
4387
|
+
* this helper as their allocation contract.
|
|
4388
|
+
*/
|
|
4389
|
+
export function compatibleLexicalOutcomes(
|
|
4390
|
+
classifier: LexicalTokenClassifier,
|
|
4391
|
+
input: string,
|
|
4392
|
+
start: number,
|
|
4393
|
+
end: number,
|
|
4394
|
+
): number[] {
|
|
4395
|
+
const ids: number[] = []
|
|
4396
|
+
let otherwise: number | undefined
|
|
4397
|
+
for (const outcome of classifier.outcomes) {
|
|
4398
|
+
const match = outcome.match
|
|
4399
|
+
if (match.kind === 'otherwise') { otherwise = outcome.id; continue }
|
|
4400
|
+
if (lexicalOutcomeMatches(match, input, start, end) && !ids.includes(outcome.id)) ids.push(outcome.id)
|
|
4401
|
+
}
|
|
4402
|
+
if (ids.length === 0 && otherwise !== undefined) ids.push(otherwise)
|
|
4403
|
+
return ids
|
|
4404
|
+
}
|
|
4405
|
+
|
|
4406
|
+
/**
|
|
4407
|
+
* Cold oracle for dispatch selection precedence. Compatible views are a set;
|
|
4408
|
+
* selected routing is still exact cases first, then matcher source order, then
|
|
4409
|
+
* otherwise, and a selected branch failure remains committed.
|
|
4410
|
+
*/
|
|
4411
|
+
export function selectedLexicalOutcome(
|
|
4412
|
+
classifier: LexicalTokenClassifier,
|
|
4413
|
+
input: string,
|
|
4414
|
+
start: number,
|
|
4415
|
+
end: number,
|
|
4416
|
+
): { route: LexicalTokenRoute; outcomeId: number } | undefined {
|
|
4417
|
+
for (const route of classifier.routes) {
|
|
4418
|
+
for (let i = 0; i < route.matches.length; i++) {
|
|
4419
|
+
const match = route.matches[i]!
|
|
4420
|
+
if (match.kind === 'otherwise' || lexicalOutcomeMatches(match, input, start, end)) {
|
|
4421
|
+
return { route, outcomeId: route.acceptedIds[i]! }
|
|
4422
|
+
}
|
|
4423
|
+
}
|
|
4424
|
+
}
|
|
4425
|
+
return undefined
|
|
4426
|
+
}
|
|
4427
|
+
|
|
124
4428
|
/** The id already assigned to this terminal, if it is one. */
|
|
125
4429
|
export function terminalId(alphabet: Alphabet, p: Combinator<unknown>): number | undefined {
|
|
126
4430
|
const key = keyOf(p._def)
|