parseman 0.61.0 → 0.61.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/backtrack.d.ts +50 -0
- package/dist/analysis/backtrack.d.ts.map +1 -0
- package/dist/analysis/diagnostics.d.ts +3 -0
- package/dist/analysis/diagnostics.d.ts.map +1 -1
- package/dist/analysis/diagnostics.js +1116 -123
- package/dist/analysis/diagnostics.js.map +3 -3
- package/dist/analysis/rebuild.d.ts +17 -1
- package/dist/analysis/rebuild.d.ts.map +1 -1
- package/dist/cli/index.js +307 -34
- package/dist/cli/index.js.map +3 -3
- package/dist/combinators/dispatch.d.ts +2 -0
- package/dist/combinators/dispatch.d.ts.map +1 -1
- package/dist/coverage.d.ts.map +1 -1
- package/dist/index.js +31 -25
- package/dist/index.js.map +1 -1
- package/dist/plugin/index.d.ts +3 -1
- package/dist/plugin/index.d.ts.map +1 -1
- package/dist/plugin/index.js +53 -25
- package/dist/plugin/index.js.map +2 -2
- package/dist/table/backtrack-probe.d.ts +41 -0
- package/dist/table/backtrack-probe.d.ts.map +1 -0
- package/dist/table/child-slots.d.ts +10 -0
- package/dist/table/child-slots.d.ts.map +1 -1
- package/dist/table/index.js +21 -15
- package/dist/table/index.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/package.json +5 -1
- package/src/analysis/backtrack.ts +249 -0
- package/src/analysis/diagnostics.ts +4 -0
- package/src/analysis/rebuild.ts +63 -11
- package/src/combinators/dispatch.ts +9 -0
- package/src/coverage.ts +2 -14
- package/src/plugin/index.ts +30 -13
- package/src/table/backtrack-probe.ts +326 -0
- package/src/table/child-slots.ts +30 -14
- package/src/version.ts +1 -1
package/dist/version.d.ts
CHANGED
|
@@ -13,5 +13,5 @@
|
|
|
13
13
|
* Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
|
|
14
14
|
* the build if this constant drifts from the published version.
|
|
15
15
|
*/
|
|
16
|
-
export declare const PARSEMAN_VERSION = "0.61.
|
|
16
|
+
export declare const PARSEMAN_VERSION = "0.61.2";
|
|
17
17
|
//# sourceMappingURL=version.d.ts.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "parseman",
|
|
3
|
-
"version": "0.61.
|
|
3
|
+
"version": "0.61.2",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": "^20.19.0 || >=22.13.0"
|
|
@@ -119,6 +119,10 @@
|
|
|
119
119
|
"size:guard:jess": "node --import tsx/esm bench/jess-grammar-size-guard.ts",
|
|
120
120
|
"vendor:jess-grammars": "node scripts/vendor-jess-grammars.mjs",
|
|
121
121
|
"size:probe": "node --import tsx/esm bench/size/probe.ts",
|
|
122
|
+
"callcount:guard": "node --import tsx/esm bench/call-count-guard.ts",
|
|
123
|
+
"callcount:baseline": "node --import tsx/esm bench/call-count-guard.ts --update",
|
|
124
|
+
"dupfn:guard": "node --import tsx/esm bench/dup-function-guard.ts",
|
|
125
|
+
"dupfn:baseline": "node --import tsx/esm bench/dup-function-guard.ts --update",
|
|
122
126
|
"choicecost:guard": "node --import tsx/esm bench/choice-cost-guard.ts",
|
|
123
127
|
"choicecost:baseline": "node --import tsx/esm bench/choice-cost-guard.ts --update",
|
|
124
128
|
"choicecost:report": "node --import tsx/esm bench/choice-cost-guard.ts --report",
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MEASURE BACKTRACKING among the terminals a parse matched.
|
|
3
|
+
*
|
|
4
|
+
* `backtrackProbe(grammar, engine)` parses with an instrumented copy of the grammar
|
|
5
|
+
* and reports, per parse, how often a matched terminal started behind the one before
|
|
6
|
+
* it and how many bytes matched twice, plus the lookahead it did (`BacktrackCounts`,
|
|
7
|
+
* `table/backtrack-probe.ts`). Each `measure` also runs the plain grammar, returns
|
|
8
|
+
* that result, and refuses to report if the instrumented parse returned anything else.
|
|
9
|
+
*
|
|
10
|
+
* Counts of zero do not show a parse wasted nothing. Failed reads, trivia re-scans,
|
|
11
|
+
* the choice predecide pretest and table `token()` bodies go uncounted; the list and
|
|
12
|
+
* an example are in `table/backtrack-probe.ts`.
|
|
13
|
+
*
|
|
14
|
+
* Three engines, one rule. `closure` and `emitted` run a rewritten copy of the
|
|
15
|
+
* table (`probeProgram`); `interpreter` runs a rebuilt copy of the combinator graph
|
|
16
|
+
* whose terminals report to the same recorder. Nothing here is reachable from a
|
|
17
|
+
* parse that did not ask for it, which is why it costs an ordinary parse nothing.
|
|
18
|
+
*/
|
|
19
|
+
import type { Combinator, ParseContext, ParseResult } from '../types.ts'
|
|
20
|
+
import { run, type RunOptions, type RunResult } from '../functional/run.ts'
|
|
21
|
+
import { digestValue } from '../oracle/digest.ts'
|
|
22
|
+
import { compileRuleMapRunnable } from '../table/compile-rule-map.ts'
|
|
23
|
+
import { expandCompact, type CompactProgram, type TableProgram } from '../table/program.ts'
|
|
24
|
+
import { AssemblyCache, tableRules } from '../table/assemble.ts'
|
|
25
|
+
import {
|
|
26
|
+
BacktrackRecorder, hasLookaround, probeCfg, probeProgram, type BacktrackCounts,
|
|
27
|
+
} from '../table/backtrack-probe.ts'
|
|
28
|
+
import { rebuildCombinator } from './rebuild.ts'
|
|
29
|
+
|
|
30
|
+
export type { BacktrackCounts } from '../table/backtrack-probe.ts'
|
|
31
|
+
|
|
32
|
+
export type BacktrackEngine = 'interpreter' | 'closure'
|
|
33
|
+
|
|
34
|
+
export type BacktrackMeasurement = {
|
|
35
|
+
readonly result: RunResult
|
|
36
|
+
readonly counts: BacktrackCounts
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type BacktrackProbe = {
|
|
40
|
+
/** The engine probed parses run on. */
|
|
41
|
+
readonly engine: { readonly ran: BacktrackEngine | 'emitted' }
|
|
42
|
+
/**
|
|
43
|
+
* Parse `input` from rule `entry`, plainly and probed, count the probed parse and
|
|
44
|
+
* return the plain parse's result. Throws if the two parses differ in outcome, value
|
|
45
|
+
* or where they stopped.
|
|
46
|
+
*
|
|
47
|
+
* Both parses get the same `options.state`, the plain one first. A grammar that
|
|
48
|
+
* writes state (a `gate()` that mutates it, say) sees the first parse's writes in the
|
|
49
|
+
* second, so the measure either throws or counts a different parse, possibly with
|
|
50
|
+
* fewer rewinds. Measure such a grammar only if resetting its state makes no difference.
|
|
51
|
+
*/
|
|
52
|
+
measure(entry: string, input: string, options?: RunOptions): BacktrackMeasurement
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
type Runnable = Parameters<typeof run>[0]
|
|
56
|
+
|
|
57
|
+
function isCombinator(x: unknown): x is Combinator<unknown> {
|
|
58
|
+
return typeof x === 'object' && x !== null && '_def' in x
|
|
59
|
+
&& typeof (x as { parse?: unknown }).parse === 'function'
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function isProgram(x: object): x is TableProgram | CompactProgram {
|
|
63
|
+
return Array.isArray((x as { code?: unknown }).code) || Array.isArray((x as { c?: unknown }).c)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The rules a `compile()` of this map would encode: combinators, minus undefined refs. */
|
|
67
|
+
function ruleEntries(grammar: Record<string, unknown>): Array<[string, Combinator<unknown>]> {
|
|
68
|
+
const out: Array<[string, Combinator<unknown>]> = []
|
|
69
|
+
for (const [name, value] of Object.entries(grammar)) {
|
|
70
|
+
if (!isCombinator(value)) continue
|
|
71
|
+
if (value._def.tag === 'lazy') {
|
|
72
|
+
try { value._def.thunk() } catch { continue }
|
|
73
|
+
}
|
|
74
|
+
out.push([name, value])
|
|
75
|
+
}
|
|
76
|
+
return out
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function outcome(r: RunResult): string {
|
|
80
|
+
return digestValue({ ok: r.ok, value: r.value, span: r.span, unconsumedFrom: r.unconsumedFrom })
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
type Grammar = Readonly<Record<string, unknown>> | TableProgram | CompactProgram
|
|
84
|
+
|
|
85
|
+
function tableOf(grammar: Grammar): TableProgram {
|
|
86
|
+
if (isProgram(grammar)) return expandCompact(grammar)
|
|
87
|
+
const refusals: string[] = []
|
|
88
|
+
const compiled = compileRuleMapRunnable(ruleEntries(grammar), { refusals })
|
|
89
|
+
if (compiled === null) throw new Error(`backtrackProbe: this grammar could not be encoded to a table: ${refusals.join('; ')}`)
|
|
90
|
+
return compiled.prog
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Probe a grammar on one engine.
|
|
95
|
+
*
|
|
96
|
+
* `grammar` is a rule map of combinators (a `rules()` or `compose()` result), or a
|
|
97
|
+
* table program for the closure engine. A rule map runs on the table through
|
|
98
|
+
* `compileRuleMapRunnable`, the lowering `compile()` uses.
|
|
99
|
+
*
|
|
100
|
+
* The emitted engine is not offered here: it needs code built from source, which
|
|
101
|
+
* at runtime only `compile()` may do (`docs/design/runtime-and-size-contract.md`,
|
|
102
|
+
* rule 1). Tests and benches reach it through `backtrackProbeAssembled` with a
|
|
103
|
+
* build-time module (`bench/emitted-assembly.ts`). It counts what the closure
|
|
104
|
+
* engine counts, because both run the same rewritten program.
|
|
105
|
+
*/
|
|
106
|
+
export function backtrackProbe(grammar: Grammar, engine: BacktrackEngine = 'closure'): BacktrackProbe {
|
|
107
|
+
const rec = new BacktrackRecorder()
|
|
108
|
+
if (engine === 'interpreter') {
|
|
109
|
+
if (isProgram(grammar)) throw new TypeError('backtrackProbe: the interpreter runs a rule map of combinators, not a table program')
|
|
110
|
+
const rules = Object.fromEntries(ruleEntries(grammar))
|
|
111
|
+
const built = new Map<string, Combinator<unknown>>()
|
|
112
|
+
const instrument = instrumentFor(rec)
|
|
113
|
+
return probe(rec, 'interpreter', entry => ruleOf(rules, entry), (entry) => {
|
|
114
|
+
let hit = built.get(entry)
|
|
115
|
+
if (hit === undefined) {
|
|
116
|
+
hit = rebuildCombinator(ruleOf(rules, entry), new Map(), { thaw: true, after: instrument }).root
|
|
117
|
+
built.set(entry, hit)
|
|
118
|
+
}
|
|
119
|
+
return hit
|
|
120
|
+
})
|
|
121
|
+
}
|
|
122
|
+
if (engine !== 'closure') throw new TypeError(`backtrackProbe: no engine ${JSON.stringify(engine)}; it runs 'closure' or 'interpreter'`)
|
|
123
|
+
const prog = tableOf(grammar)
|
|
124
|
+
const base = tableRules({ ...prog, asm: [] })
|
|
125
|
+
const rules = tableRules(probeProgram(prog, rec))
|
|
126
|
+
return probe(rec, 'closure', entry => ruleOf(base, entry), entry => ruleOf(rules, entry))
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* For tests and benches, not `parseman/diagnostics`: the probe on the emitted
|
|
131
|
+
* engine. `assemble` hands the rewritten program back with a pre-compiled
|
|
132
|
+
* assembly for `probeCfg(program)`, built as a module the way a macro build ships
|
|
133
|
+
* one. Nothing here turns a string into code, and a program with no assembly is
|
|
134
|
+
* refused, because the assembler would.
|
|
135
|
+
*/
|
|
136
|
+
export async function backtrackProbeAssembled(
|
|
137
|
+
grammar: Grammar,
|
|
138
|
+
assemble: (probed: TableProgram) => Promise<TableProgram>,
|
|
139
|
+
): Promise<BacktrackProbe> {
|
|
140
|
+
const rec = new BacktrackRecorder()
|
|
141
|
+
const prog = tableOf(grammar)
|
|
142
|
+
const base = tableRules({ ...prog, asm: [] })
|
|
143
|
+
const assembled = await assemble(probeProgram(prog, rec))
|
|
144
|
+
// A program with no inventory would make the assembler build the engine from source.
|
|
145
|
+
if (assembled.asm === undefined) throw new TypeError('backtrackProbeAssembled: `assemble` returned a program with no pre-compiled assembly')
|
|
146
|
+
// Ask the assembler which engine it built, rather than assume the factory was taken.
|
|
147
|
+
const refusal = new AssemblyCache(assembled).for(probeCfg(assembled)).emitRefusal
|
|
148
|
+
if (refusal !== undefined) throw new Error(`backtrackProbeAssembled: the assembly was not used: ${refusal}`)
|
|
149
|
+
const rules = tableRules(assembled)
|
|
150
|
+
const emitted = probe(rec, 'emitted', entry => ruleOf(base, entry), entry => ruleOf(rules, entry))
|
|
151
|
+
return {
|
|
152
|
+
engine: emitted.engine,
|
|
153
|
+
measure(entry, input, options = {}) {
|
|
154
|
+
// The assembly serves `probeCfg`'s option set alone. An option that selects
|
|
155
|
+
// another one (tolerant, completions, a build host…) would run closures under
|
|
156
|
+
// an `emitted` label, so only the parse state may be passed.
|
|
157
|
+
const other = Object.keys(options).filter(k => k !== 'state')
|
|
158
|
+
if (other.length > 0) throw new TypeError(`backtrackProbeAssembled: the emitted probe runs a plain parse; it takes no ${other.join(', ')}`)
|
|
159
|
+
return emitted.measure(entry, input, options)
|
|
160
|
+
},
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function probe(
|
|
165
|
+
rec: BacktrackRecorder,
|
|
166
|
+
ran: BacktrackProbe['engine']['ran'],
|
|
167
|
+
plain: (entry: string) => Runnable,
|
|
168
|
+
probed: (entry: string) => Runnable,
|
|
169
|
+
): BacktrackProbe {
|
|
170
|
+
return {
|
|
171
|
+
engine: { ran },
|
|
172
|
+
measure(entry, input, options = {}) {
|
|
173
|
+
const result = run(plain(entry), input, options)
|
|
174
|
+
rec.begin(input)
|
|
175
|
+
const probedResult = run(probed(entry), input, options)
|
|
176
|
+
const counts = rec.end()
|
|
177
|
+
if (outcome(probedResult) !== outcome(result)) {
|
|
178
|
+
throw new Error(`backtrackProbe: the probed ${ran} parse of rule ${JSON.stringify(entry)} differs from the plain one; its counts would describe another parse`)
|
|
179
|
+
}
|
|
180
|
+
return { result, counts }
|
|
181
|
+
},
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function ruleOf<T>(rules: Readonly<Record<string, T>>, entry: string): T {
|
|
186
|
+
if (!Object.hasOwn(rules, entry)) throw new Error(`backtrackProbe: the grammar has no rule ${JSON.stringify(entry)}`)
|
|
187
|
+
return rules[entry]!
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
type Parse = (input: string, pos: number, ctx: ParseContext) => ParseResult<unknown>
|
|
191
|
+
|
|
192
|
+
function wrapped(out: Combinator<unknown>, parse: Parse): Combinator<unknown> {
|
|
193
|
+
const w: Combinator<unknown> = { _tag: out._tag, _meta: out._meta, _def: out._def, parse }
|
|
194
|
+
const name = (out as { _ruleName?: string })._ruleName
|
|
195
|
+
if (name !== undefined) (w as { _ruleName?: string })._ruleName = name
|
|
196
|
+
return w
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* The interpreter's half of the rewrite `probeProgram` makes to a table: terminals
|
|
201
|
+
* report their span, a lookahead marks its extent, and a scan's own reads stay
|
|
202
|
+
* inside it. The rebuilt node is wrapped, never patched, so the grammar the caller
|
|
203
|
+
* holds is untouched.
|
|
204
|
+
*/
|
|
205
|
+
function instrumentFor(rec: BacktrackRecorder): (original: Combinator<unknown>, rebuilt: Combinator<unknown>) => Combinator<unknown> {
|
|
206
|
+
return (_original, out) => {
|
|
207
|
+
const d = out._def
|
|
208
|
+
switch (d.tag) {
|
|
209
|
+
case 'literal': case 'keywords':
|
|
210
|
+
return wrapped(out, (input, pos, ctx) => {
|
|
211
|
+
const r = out.parse(input, pos, ctx)
|
|
212
|
+
if (r.ok) rec.consume(r.span.start, r.span.end)
|
|
213
|
+
return r
|
|
214
|
+
})
|
|
215
|
+
case 'regex': {
|
|
216
|
+
const re = new RegExp(d.source, d.flags)
|
|
217
|
+
if (!hasLookaround(re)) {
|
|
218
|
+
return wrapped(out, (input, pos, ctx) => {
|
|
219
|
+
const r = out.parse(input, pos, ctx)
|
|
220
|
+
if (r.ok) rec.consume(r.span.start, r.span.end)
|
|
221
|
+
return r
|
|
222
|
+
})
|
|
223
|
+
}
|
|
224
|
+
return wrapped(out, (input, pos, ctx) => {
|
|
225
|
+
rec.regexAttempt()
|
|
226
|
+
const r = out.parse(input, pos, ctx)
|
|
227
|
+
if (r.ok) rec.regexMatch(re, r.span.start, r.span.end)
|
|
228
|
+
return r
|
|
229
|
+
})
|
|
230
|
+
}
|
|
231
|
+
case 'scanTo': case 'lines': case 'columns': case 'recover': case 'unknown':
|
|
232
|
+
return wrapped(out, (input, pos, ctx) => {
|
|
233
|
+
rec.enterOpaque()
|
|
234
|
+
let r: ParseResult<unknown>
|
|
235
|
+
try { r = out.parse(input, pos, ctx) } finally { rec.exitOpaque() }
|
|
236
|
+
if (r.ok) rec.consume(r.span.start, r.span.end)
|
|
237
|
+
return r
|
|
238
|
+
})
|
|
239
|
+
case 'not': case 'peek':
|
|
240
|
+
return wrapped(out, (input, pos, ctx) => {
|
|
241
|
+
rec.enterLookahead()
|
|
242
|
+
try { return out.parse(input, pos, ctx) } finally { rec.exitLookahead() }
|
|
243
|
+
})
|
|
244
|
+
default:
|
|
245
|
+
return out
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
@@ -30,4 +30,8 @@ export { measureChoiceCost, armFirstSets } from './corpus.ts'
|
|
|
30
30
|
export type { CorpusSample, CorpusSite, ChoiceCorpusCost, ArmCorpusCost } from './corpus.ts'
|
|
31
31
|
|
|
32
32
|
export { rebuildCombinator } from './rebuild.ts'
|
|
33
|
+
export type { RebuildOptions } from './rebuild.ts'
|
|
34
|
+
|
|
35
|
+
export { backtrackProbe } from './backtrack.ts'
|
|
36
|
+
export type { BacktrackProbe, BacktrackEngine, BacktrackMeasurement, BacktrackCounts } from './backtrack.ts'
|
|
33
37
|
export type { RebuildResult, FrozenSubtree } from './rebuild.ts'
|
package/src/analysis/rebuild.ts
CHANGED
|
@@ -53,12 +53,17 @@ import { token, leaf, sourceLeaf } from '../combinators/token.ts'
|
|
|
53
53
|
import { expect } from '../combinators/expect.ts'
|
|
54
54
|
import { adjacent, notAdjacent } from '../combinators/adjacency.ts'
|
|
55
55
|
import { parser } from '../combinators/grammar.ts'
|
|
56
|
+
import { dispatch, dispatchMatcher, otherwise, routed, when, type DispatchArm } from '../combinators/dispatch.ts'
|
|
57
|
+
import { withCtx } from '../combinators/withCtx.ts'
|
|
56
58
|
|
|
57
59
|
/** A node kind reused verbatim: no public factory reconstructs it without loss. */
|
|
58
60
|
const FROZEN_TAGS: ReadonlySet<string> = new Set([
|
|
59
61
|
'dispatch', 'recover', 'scanTo', 'guard', 'withCtx', 'routed', 'trivia', 'unknown', 'lines', 'columns',
|
|
60
62
|
])
|
|
61
63
|
|
|
64
|
+
/** The frozen kinds `RebuildOptions.thaw` rebuilds. */
|
|
65
|
+
const THAWED_TAGS: ReadonlySet<string> = new Set(['dispatch', 'withCtx', 'routed', 'guard'])
|
|
66
|
+
|
|
62
67
|
/** A subtree that was reused verbatim instead of rebuilt. */
|
|
63
68
|
export type FrozenSubtree = {
|
|
64
69
|
tag: string
|
|
@@ -117,6 +122,23 @@ function subtreeContains(p: Combinator<unknown>, targets: ReadonlySet<Combinator
|
|
|
117
122
|
return hit
|
|
118
123
|
}
|
|
119
124
|
|
|
125
|
+
export type RebuildOptions = {
|
|
126
|
+
/**
|
|
127
|
+
* Also rebuild `dispatch`, a gated `choice`, `withCtx` and `routed` instead of
|
|
128
|
+
* freezing them. Each is reconstructed from its own def fields — the arms, gates,
|
|
129
|
+
* matchers and `extra` the grammar built it with. Off for `proposeFixes()`, whose
|
|
130
|
+
* reports name the frozen kinds; on for an instrumented rebuild, which must reach
|
|
131
|
+
* every terminal (`backtrack.ts`).
|
|
132
|
+
*/
|
|
133
|
+
readonly thaw?: boolean
|
|
134
|
+
/**
|
|
135
|
+
* Called with each node and what it was rebuilt to (itself, for a frozen or
|
|
136
|
+
* childless one), and its result is used in its place. Not called for a
|
|
137
|
+
* replacement or a `lazy` slot.
|
|
138
|
+
*/
|
|
139
|
+
readonly after?: (original: Combinator<unknown>, rebuilt: Combinator<unknown>) => Combinator<unknown>
|
|
140
|
+
}
|
|
141
|
+
|
|
120
142
|
/**
|
|
121
143
|
* Rebuild `root`, replacing every combinator that is a key of `replacements` with its
|
|
122
144
|
* value. The replacement is spliced WITHOUT descending into the original target.
|
|
@@ -124,7 +146,10 @@ function subtreeContains(p: Combinator<unknown>, targets: ReadonlySet<Combinator
|
|
|
124
146
|
export function rebuildCombinator(
|
|
125
147
|
root: Combinator<unknown>,
|
|
126
148
|
replacements: ReadonlyMap<Combinator<unknown>, Combinator<unknown>>,
|
|
149
|
+
options: RebuildOptions = {},
|
|
127
150
|
): RebuildResult {
|
|
151
|
+
const thaw = options.thaw === true
|
|
152
|
+
const after = options.after ?? ((_p: Combinator<unknown>, out: Combinator<unknown>) => out)
|
|
128
153
|
const memo = new Map<Combinator<unknown>, Combinator<unknown>>()
|
|
129
154
|
const frozenKeys = new Set<string>()
|
|
130
155
|
const frozen: FrozenSubtree[] = []
|
|
@@ -141,12 +166,13 @@ export function rebuildCombinator(
|
|
|
141
166
|
if (d === undefined || typeof d !== 'object') { memo.set(p, p); return p }
|
|
142
167
|
const here = (p as { _ruleName?: string })._ruleName ?? rule
|
|
143
168
|
|
|
144
|
-
if (FROZEN_TAGS.has(d.tag)) {
|
|
169
|
+
if (FROZEN_TAGS.has(d.tag) && !(thaw && THAWED_TAGS.has(d.tag))) {
|
|
145
170
|
const key = JSON.stringify([here, d.tag])
|
|
146
171
|
if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: d.tag, rule: here }) }
|
|
147
172
|
for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
|
|
148
|
-
|
|
149
|
-
|
|
173
|
+
const kept = after(p, p)
|
|
174
|
+
memo.set(p, kept)
|
|
175
|
+
return kept
|
|
150
176
|
}
|
|
151
177
|
|
|
152
178
|
// A ref/lazy is the only place a cycle can close. Publish an EMPTY slot into the
|
|
@@ -188,12 +214,21 @@ export function rebuildCombinator(
|
|
|
188
214
|
// arity (it takes GatedArm objects the grammar built), and the whole point of
|
|
189
215
|
// rebuilding a choice is that its strategy is recomputed — which the gated path
|
|
190
216
|
// suppresses anyway. Freeze it rather than rebuild it wrong.
|
|
217
|
+
if (d.gates.some(g => g !== null) && thaw) {
|
|
218
|
+
const unsafeChoice = choice as unknown as (...items: unknown[]) => Combinator<unknown>
|
|
219
|
+
out = unsafeChoice(...d.parsers.map((arm, i) => {
|
|
220
|
+
const gate = d.gates[i]
|
|
221
|
+
return gate ? { combinator: one(arm), gate } : one(arm)
|
|
222
|
+
}))
|
|
223
|
+
break
|
|
224
|
+
}
|
|
191
225
|
if (d.gates.some(g => g !== null)) {
|
|
192
226
|
const key = JSON.stringify([here, "choice(gated)"])
|
|
193
227
|
if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: 'choice(gated)', rule: here }) }
|
|
194
228
|
for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
|
|
195
|
-
|
|
196
|
-
|
|
229
|
+
const kept = after(p, p)
|
|
230
|
+
memo.set(p, kept)
|
|
231
|
+
return kept
|
|
197
232
|
}
|
|
198
233
|
const kids = d.parsers.map(one) as [Combinator<unknown>, ...Combinator<unknown>[]]
|
|
199
234
|
out = choice(...kids); break
|
|
@@ -250,8 +285,9 @@ export function rebuildCombinator(
|
|
|
250
285
|
const key = `${here} node(untyped+build+opts)`
|
|
251
286
|
if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: 'node(untyped+build+opts)', rule: here }) }
|
|
252
287
|
for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
|
|
253
|
-
|
|
254
|
-
|
|
288
|
+
const kept = after(p, p)
|
|
289
|
+
memo.set(p, kept)
|
|
290
|
+
return kept
|
|
255
291
|
}
|
|
256
292
|
out = d.type === undefined
|
|
257
293
|
? (d.build === undefined ? node(one(d.parser), opts) : node(one(d.parser), d.build))
|
|
@@ -259,6 +295,20 @@ export function rebuildCombinator(
|
|
|
259
295
|
if (d.buildRawUnused === true && out._def.tag === 'node') out._def.buildRawUnused = true
|
|
260
296
|
break
|
|
261
297
|
}
|
|
298
|
+
case 'dispatch': {
|
|
299
|
+
const arms: DispatchArm<unknown>[] = d.cases.map(e =>
|
|
300
|
+
when(e.keys, one(e.parser), e.caseInsensitive ? { caseInsensitive: true } : undefined))
|
|
301
|
+
for (const e of d.matchers ?? []) {
|
|
302
|
+
arms.push(when(dispatchMatcher(e), one(e.parser), e.caseInsensitive ? { caseInsensitive: true } : undefined))
|
|
303
|
+
}
|
|
304
|
+
if (d.otherwise) arms.push(otherwise(one(d.otherwise), d.otherwiseCommit === false ? { commit: false } : undefined))
|
|
305
|
+
const unsafeDispatch = dispatch as (selector: Combinator<string>, ...items: DispatchArm<unknown>[]) => Combinator<unknown>
|
|
306
|
+
out = unsafeDispatch(one(d.selector) as Combinator<string>, ...arms)
|
|
307
|
+
break
|
|
308
|
+
}
|
|
309
|
+
case 'withCtx': out = withCtx(d.extra, one(d.parser)); break
|
|
310
|
+
case 'routed': out = d.fallback === undefined ? routed() : routed(one(d.fallback)); break
|
|
311
|
+
case 'guard': out = p; break
|
|
262
312
|
case 'grammar': {
|
|
263
313
|
const inner = one(d.parser)
|
|
264
314
|
out = parser({
|
|
@@ -273,13 +323,15 @@ export function rebuildCombinator(
|
|
|
273
323
|
const key = JSON.stringify([here, (d as { tag: string }).tag])
|
|
274
324
|
if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: (d as { tag: string }).tag, rule: here }) }
|
|
275
325
|
for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
|
|
276
|
-
|
|
277
|
-
|
|
326
|
+
const kept = after(p, p)
|
|
327
|
+
memo.set(p, kept)
|
|
328
|
+
return kept
|
|
278
329
|
}
|
|
279
330
|
}
|
|
280
331
|
carryStamps(p, out)
|
|
281
|
-
|
|
282
|
-
|
|
332
|
+
const done = after(p, out)
|
|
333
|
+
memo.set(p, done)
|
|
334
|
+
return done
|
|
283
335
|
}
|
|
284
336
|
|
|
285
337
|
const rebuilt = go(root, (root as { _ruleName?: string })._ruleName ?? '<entry>')
|
|
@@ -180,6 +180,15 @@ export function matches(pattern: RegExp): DispatchStringMatcher {
|
|
|
180
180
|
return { kind: 'matches', value: pattern.source, flags: pattern.flags }
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
+
/** The matcher a `dispatch()` def records for a case, as the builder that made it. */
|
|
184
|
+
export function dispatchMatcher(entry: DispatchMatcherCase): DispatchStringMatcher {
|
|
185
|
+
switch (entry.kind) {
|
|
186
|
+
case 'startsWith': return startsWith(entry.value)
|
|
187
|
+
case 'endsWith': return endsWith(entry.value)
|
|
188
|
+
case 'matches': return matches(new RegExp(entry.value, entry.flags))
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
183
192
|
export function when<T>(
|
|
184
193
|
key: DispatchWhenKey,
|
|
185
194
|
parser: Combinator<T>,
|
package/src/coverage.ts
CHANGED
|
@@ -2,9 +2,9 @@ import { lines } from './combinators/lines.ts'
|
|
|
2
2
|
import { run, type RunOptions, type RunResult, type Runnable } from './functional/run.ts'
|
|
3
3
|
import { GRAMMAR_COVERAGE_DEFINITIONS } from './grammar-metadata.ts'
|
|
4
4
|
export { GRAMMAR_COVERAGE_DEFINITIONS } from './grammar-metadata.ts'
|
|
5
|
-
import type { Combinator, DispatchCase,
|
|
5
|
+
import type { Combinator, DispatchCase, ParseContext, ParseResult } from './types.ts'
|
|
6
6
|
import { choice } from './combinators/choice.ts'
|
|
7
|
-
import { dispatch,
|
|
7
|
+
import { dispatch, dispatchMatcher, otherwise, when, type DispatchArm } from './combinators/dispatch.ts'
|
|
8
8
|
import { attempt } from './combinators/attempt.ts'
|
|
9
9
|
import { getCoreLiteralValue } from './combinators/choice.ts'
|
|
10
10
|
import { not } from './combinators/not.ts'
|
|
@@ -24,18 +24,6 @@ import { buildGrammarPlan, type GrammarCoverageDefinition, type GrammarCoverageP
|
|
|
24
24
|
|
|
25
25
|
export type { GrammarCoverageDefinition } from './compiler/grammar-coverage-ids.ts'
|
|
26
26
|
|
|
27
|
-
function dispatchMatcher(entry: DispatchMatcherCase): DispatchStringMatcher {
|
|
28
|
-
switch (entry.kind) {
|
|
29
|
-
case 'startsWith':
|
|
30
|
-
return startsWith(entry.value)
|
|
31
|
-
case 'endsWith':
|
|
32
|
-
return endsWith(entry.value)
|
|
33
|
-
case 'matches':
|
|
34
|
-
return matches(new RegExp(entry.value, entry.flags))
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
39
27
|
export type GrammarCoverageSnapshot = {
|
|
40
28
|
definitions: readonly GrammarCoverageDefinition[]
|
|
41
29
|
hits: readonly string[]
|
package/src/plugin/index.ts
CHANGED
|
@@ -184,7 +184,9 @@ export type AssemblyProfile = {
|
|
|
184
184
|
/** The rule each corpus file is parsed from, e.g. `'Stylesheet'`. A grammar
|
|
185
185
|
* without this rule keeps the static ranking. */
|
|
186
186
|
readonly entry: string
|
|
187
|
-
/** Paths of representative inputs.
|
|
187
|
+
/** Paths of representative inputs. At least one must have content, and each that
|
|
188
|
+
* does has to parse in full without recovery, or the build fails. A file with only
|
|
189
|
+
* whitespace counts as empty. */
|
|
188
190
|
readonly files: readonly string[]
|
|
189
191
|
}
|
|
190
192
|
|
|
@@ -2376,29 +2378,44 @@ function transformMacroImpl(
|
|
|
2376
2378
|
return sites
|
|
2377
2379
|
}
|
|
2378
2380
|
/** Measured site calls over the optional corpus, or `undefined` for the static
|
|
2379
|
-
* ranking. A
|
|
2381
|
+
* ranking. A corpus that cannot run fails the build: a static ranking standing in
|
|
2382
|
+
* for the measured one configured is a slower parser that nothing reports. */
|
|
2380
2383
|
const profileOf = (compiled: CompiledRuleMapTable): Int32Array | undefined => {
|
|
2381
2384
|
// Only the default AST/no-lines table carries an assembly, so only it is profiled.
|
|
2382
2385
|
if (assemblyProfile === undefined || compiled.hostMode !== 'ast' || compiled.prog.lines === 1
|
|
2383
2386
|
|| !(assemblyProfile.entry in compiled.prog.rules)) return undefined
|
|
2384
|
-
|
|
2387
|
+
const refuse = (why: string): never => { throw new Error(`${id} — assemblyProfile: ${why}`) }
|
|
2388
|
+
if (corpusTexts(assemblyProfile).every(text => text.trim() === '')) refuse('`files` has no non-empty file')
|
|
2389
|
+
let thrower: unknown
|
|
2385
2390
|
// A builder re-lowered from carried IR is inert at build time; for COUNTING
|
|
2386
2391
|
// only, a plain node stands in for it, so the corpus can run past it.
|
|
2387
2392
|
const fns = compiled.prog.fns.map(fn => {
|
|
2388
2393
|
const type = (fn as { [INERT_BUILDER]?: string } | null)?.[INERT_BUILDER]
|
|
2389
|
-
|
|
2390
|
-
|
|
2394
|
+
if (type !== undefined) return (children: unknown, _fields: unknown, span: unknown) => ({ type, span, children })
|
|
2395
|
+
return typeof fn !== 'function' ? fn : new Proxy(fn, {
|
|
2396
|
+
apply(target, self, args) {
|
|
2397
|
+
try { return Reflect.apply(target, self, args) as unknown } catch (e) { thrower ??= target; throw e }
|
|
2398
|
+
},
|
|
2399
|
+
})
|
|
2391
2400
|
})
|
|
2392
|
-
|
|
2401
|
+
return countSiteVisits(compiled.prog, rules => {
|
|
2393
2402
|
const entry = rules[assemblyProfile.entry]!
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2403
|
+
corpusTexts(assemblyProfile).forEach((text, i) => {
|
|
2404
|
+
const file = assemblyProfile.files[i]!
|
|
2405
|
+
if (text.trim() === '') return
|
|
2406
|
+
let result
|
|
2407
|
+
try { result = run(entry, text) } catch (e) {
|
|
2408
|
+
const source = String(thrower).replace(/\s+/g, ' ').slice(0, 200)
|
|
2409
|
+
throw new Error(`${id} — assemblyProfile: ${thrower === undefined ? 'the parse' : `gate or reducer \`${source}\``} threw on ${file}: ${String(e)}`, { cause: e })
|
|
2410
|
+
}
|
|
2411
|
+
if (!result.ok || result.unconsumedFrom !== null) {
|
|
2412
|
+
refuse(`${file} does not parse from rule "${assemblyProfile.entry}": it stops at offset ${result.ok ? result.unconsumedFrom : result.span.end}`)
|
|
2413
|
+
}
|
|
2414
|
+
if (result.errors.length > 0) {
|
|
2415
|
+
refuse(`${file} parses only by recovering from ${result.errors.length} error(s), first at offset ${result.errors[0]!.span.start}`)
|
|
2416
|
+
}
|
|
2417
|
+
})
|
|
2397
2418
|
}, fns)
|
|
2398
|
-
if (parsed > 0) return counts
|
|
2399
|
-
// Not `warn()`: nothing fell back to the interpreter — the build is only untuned.
|
|
2400
|
-
warnings.push(`${id} — assemblyProfile: no corpus file parsed from rule "${assemblyProfile.entry}"; using the static ranking`)
|
|
2401
|
-
return undefined
|
|
2402
2419
|
}
|
|
2403
2420
|
/** The fused table's replacement, with its selective default assembly if any. */
|
|
2404
2421
|
const assembledReplacement = (compiled: CompiledRuleMapTable, metadata: string, select: ReadonlySet<number>): string =>
|