parseman 0.46.0 → 0.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/analysis/choice-cost-render.d.ts +12 -4
- package/dist/analysis/choice-cost-render.d.ts.map +1 -1
- package/dist/analysis/choice-cost.d.ts +2 -2
- package/dist/analysis/choice-cost.d.ts.map +1 -1
- package/dist/analysis/commitment.d.ts +131 -0
- package/dist/analysis/commitment.d.ts.map +1 -0
- package/dist/analysis/diagnostics.cjs +10339 -7787
- package/dist/analysis/diagnostics.cjs.map +3 -4
- package/dist/analysis/diagnostics.d.ts +2 -2
- package/dist/analysis/diagnostics.js +10339 -7787
- package/dist/analysis/diagnostics.js.map +3 -4
- package/dist/analysis/duplication.d.ts.map +1 -1
- package/dist/analysis/fix.d.ts +3 -3
- package/dist/analysis/fix.d.ts.map +1 -1
- package/dist/analysis/format-number.d.ts +23 -0
- package/dist/analysis/format-number.d.ts.map +1 -0
- package/dist/analysis/gating.d.ts +39 -2
- package/dist/analysis/gating.d.ts.map +1 -1
- package/dist/analysis/rebuild.d.ts.map +1 -1
- package/dist/analysis/terminal.d.ts +5 -2
- package/dist/analysis/terminal.d.ts.map +1 -1
- package/dist/cli/index.cjs +10924 -8506
- package/dist/cli/index.cjs.map +3 -4
- package/dist/cli/index.js +10924 -8506
- package/dist/cli/index.js.map +3 -4
- package/dist/combinators/adjacency.d.ts +100 -0
- package/dist/combinators/adjacency.d.ts.map +1 -0
- package/dist/combinators/choice.d.ts.map +1 -1
- package/dist/combinators/completions.d.ts.map +1 -1
- package/dist/combinators/dispatch.d.ts +8 -1
- package/dist/combinators/dispatch.d.ts.map +1 -1
- package/dist/combinators/expect.d.ts.map +1 -1
- package/dist/combinators/first-set.d.ts.map +1 -1
- package/dist/combinators/gate.d.ts.map +1 -1
- package/dist/combinators/grammar.d.ts +22 -0
- package/dist/combinators/grammar.d.ts.map +1 -1
- package/dist/combinators/literal.d.ts +8 -0
- package/dist/combinators/literal.d.ts.map +1 -1
- package/dist/combinators/map.d.ts +0 -1
- package/dist/combinators/map.d.ts.map +1 -1
- package/dist/combinators/node.d.ts +13 -3
- package/dist/combinators/node.d.ts.map +1 -1
- package/dist/combinators/parser.d.ts +1 -1
- package/dist/combinators/parser.d.ts.map +1 -1
- package/dist/combinators/ref.d.ts.map +1 -1
- package/dist/combinators/regex.d.ts +1 -16
- package/dist/combinators/regex.d.ts.map +1 -1
- package/dist/combinators/repeat.d.ts +30 -2
- package/dist/combinators/repeat.d.ts.map +1 -1
- package/dist/combinators/scanTo.d.ts +71 -1
- package/dist/combinators/scanTo.d.ts.map +1 -1
- package/dist/combinators/sequence.d.ts.map +1 -1
- package/dist/combinators/token.d.ts.map +1 -1
- package/dist/combinators/trivia-boundary.d.ts +26 -0
- package/dist/combinators/trivia-boundary.d.ts.map +1 -0
- package/dist/combinators/trivia-skip.d.ts +45 -1
- package/dist/combinators/trivia-skip.d.ts.map +1 -1
- package/dist/combinators/withCtx.d.ts.map +1 -1
- package/dist/compiler/compile-linkable-table.d.ts +104 -0
- package/dist/compiler/compile-linkable-table.d.ts.map +1 -0
- package/dist/compiler/degradation.d.ts +0 -2
- package/dist/compiler/degradation.d.ts.map +1 -1
- package/dist/compiler/fields.d.ts +13 -0
- package/dist/compiler/fields.d.ts.map +1 -1
- package/dist/compiler/grammar-coverage-ids.d.ts +5 -5
- package/dist/compiler/grammar-coverage-ids.d.ts.map +1 -1
- package/dist/compiler/ir-serialize.d.ts.map +1 -1
- package/dist/compiler/linker.d.ts +47 -53
- package/dist/compiler/linker.d.ts.map +1 -1
- package/dist/compiler/token-alphabet.d.ts +81 -0
- package/dist/compiler/token-alphabet.d.ts.map +1 -0
- package/dist/compiler/token-dispatch.d.ts +95 -0
- package/dist/compiler/token-dispatch.d.ts.map +1 -0
- package/dist/compiler/token-scanner.d.ts +53 -0
- package/dist/compiler/token-scanner.d.ts.map +1 -0
- package/dist/compiler/value-usage.d.ts.map +1 -1
- package/dist/coverage.d.ts +1 -1
- package/dist/coverage.d.ts.map +1 -1
- package/dist/cst/capture-buffer.d.ts +35 -0
- package/dist/cst/capture-buffer.d.ts.map +1 -1
- package/dist/cst/reflection-attach.d.ts +8 -0
- package/dist/cst/reflection-attach.d.ts.map +1 -0
- package/dist/cst/reflection-symbols.d.ts +10 -0
- package/dist/cst/reflection-symbols.d.ts.map +1 -0
- package/dist/cst/reflection.d.ts +2 -4
- package/dist/cst/reflection.d.ts.map +1 -1
- package/dist/cst/root-trivia-scope.d.ts +26 -0
- package/dist/cst/root-trivia-scope.d.ts.map +1 -0
- package/dist/cst/trivia-charscan.d.ts +18 -0
- package/dist/cst/trivia-charscan.d.ts.map +1 -0
- package/dist/cst/trivia-kinds.d.ts +19 -0
- package/dist/cst/trivia-kinds.d.ts.map +1 -1
- package/dist/functional/doc.d.ts.map +1 -1
- package/dist/functional/run-tabled.d.ts +12 -0
- package/dist/functional/run-tabled.d.ts.map +1 -0
- package/dist/functional/run.d.ts +25 -42
- package/dist/functional/run.d.ts.map +1 -1
- package/dist/grammar-metadata.d.ts +3 -0
- package/dist/grammar-metadata.d.ts.map +1 -0
- package/dist/index.cjs +9290 -7181
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +18 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9300 -7191
- package/dist/index.js.map +3 -4
- package/dist/language-service/index.cjs +313 -121
- package/dist/language-service/index.cjs.map +3 -4
- package/dist/language-service/index.js +313 -121
- package/dist/language-service/index.js.map +3 -4
- package/dist/oracle/index.cjs.map +0 -1
- package/dist/oracle/index.js.map +0 -1
- package/dist/parse-context.d.ts +49 -0
- package/dist/parse-context.d.ts.map +1 -0
- package/dist/plugin/evaluator.d.ts +40 -1
- package/dist/plugin/evaluator.d.ts.map +1 -1
- package/dist/plugin/index.cjs +11175 -9534
- package/dist/plugin/index.cjs.map +3 -4
- package/dist/plugin/index.d.ts.map +1 -1
- package/dist/plugin/index.js +11175 -9534
- package/dist/plugin/index.js.map +3 -4
- package/dist/regex/classes.d.ts +28 -0
- package/dist/regex/classes.d.ts.map +1 -1
- package/dist/regex/first-set.d.ts +14 -0
- package/dist/regex/first-set.d.ts.map +1 -1
- package/dist/run/index.cjs +136 -51
- package/dist/run/index.cjs.map +2 -3
- package/dist/run/index.js +136 -51
- package/dist/run/index.js.map +2 -3
- package/dist/spec/index.cjs +2934 -1611
- package/dist/spec/index.cjs.map +3 -4
- package/dist/spec/index.js +2934 -1611
- package/dist/spec/index.js.map +3 -4
- package/dist/spec/model.d.ts +8 -0
- package/dist/spec/model.d.ts.map +1 -1
- package/dist/table/assemble.d.ts +307 -0
- package/dist/table/assemble.d.ts.map +1 -0
- package/dist/table/cell.d.ts +42 -0
- package/dist/table/cell.d.ts.map +1 -0
- package/dist/table/child-slots.d.ts +30 -0
- package/dist/table/child-slots.d.ts.map +1 -0
- package/dist/table/compile-rule-map.d.ts +136 -0
- package/dist/table/compile-rule-map.d.ts.map +1 -0
- package/dist/table/compile.d.ts +56 -0
- package/dist/table/compile.d.ts.map +1 -0
- package/dist/table/duplication-hook.d.ts +32 -0
- package/dist/table/duplication-hook.d.ts.map +1 -0
- package/dist/table/emit-assembly.d.ts +139 -0
- package/dist/table/emit-assembly.d.ts.map +1 -0
- package/dist/table/emit.d.ts +113 -0
- package/dist/table/emit.d.ts.map +1 -0
- package/dist/table/encode-baseline.d.ts +16 -0
- package/dist/table/encode-baseline.d.ts.map +1 -0
- package/dist/table/encode.d.ts +71 -0
- package/dist/table/encode.d.ts.map +1 -0
- package/dist/table/exec-baseline.d.ts +9 -0
- package/dist/table/exec-baseline.d.ts.map +1 -0
- package/dist/table/exec.d.ts +72 -0
- package/dist/table/exec.d.ts.map +1 -0
- package/dist/table/fold.d.ts +25 -0
- package/dist/table/fold.d.ts.map +1 -0
- package/dist/table/index.cjs +11463 -0
- package/dist/table/index.cjs.map +6 -0
- package/dist/table/index.d.ts +100 -0
- package/dist/table/index.d.ts.map +1 -0
- package/dist/table/index.js +11440 -0
- package/dist/table/index.js.map +6 -0
- package/dist/table/inspect.d.ts +17 -0
- package/dist/table/inspect.d.ts.map +1 -0
- package/dist/table/ops.d.ts +453 -0
- package/dist/table/ops.d.ts.map +1 -0
- package/dist/table/program.d.ts +496 -0
- package/dist/table/program.d.ts.map +1 -0
- package/dist/table/run-support.d.ts +38 -0
- package/dist/table/run-support.d.ts.map +1 -0
- package/dist/{compiler/scannable-run.d.ts → table/scan-shapes.d.ts} +14 -13
- package/dist/table/scan-shapes.d.ts.map +1 -0
- package/dist/table/site-labels.d.ts +58 -0
- package/dist/table/site-labels.d.ts.map +1 -0
- package/dist/table/stamp.d.ts +70 -0
- package/dist/table/stamp.d.ts.map +1 -0
- package/dist/types.d.ts +80 -24
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/package.json +26 -2
- package/src/analysis/choice-cost-gate.ts +359 -0
- package/src/analysis/choice-cost-render.ts +293 -0
- package/src/analysis/choice-cost.ts +1008 -0
- package/src/analysis/commitment.ts +557 -0
- package/src/analysis/corpus.ts +130 -0
- package/src/analysis/diagnose-render.ts +625 -0
- package/src/analysis/diagnose.ts +317 -0
- package/src/analysis/diagnostics.ts +33 -0
- package/src/analysis/duplication.ts +1955 -0
- package/src/analysis/fix-render.ts +217 -0
- package/src/analysis/fix.ts +669 -0
- package/src/analysis/format-number.ts +31 -0
- package/src/analysis/gating.ts +792 -0
- package/src/analysis/grammar.ts +72 -0
- package/src/analysis/rebuild.ts +287 -0
- package/src/analysis/terminal.ts +336 -0
- package/src/cli/index.ts +413 -0
- package/src/combinators/adjacency.ts +171 -0
- package/src/combinators/attempt.ts +41 -0
- package/src/combinators/case-fold.ts +90 -0
- package/src/combinators/choice.ts +466 -0
- package/src/combinators/completions.ts +64 -0
- package/src/combinators/dispatch.ts +358 -0
- package/src/combinators/expect.ts +159 -0
- package/src/combinators/first-set.ts +348 -0
- package/src/combinators/gate.ts +50 -0
- package/src/combinators/grammar.ts +272 -0
- package/src/combinators/keywords.ts +145 -0
- package/src/combinators/literal.ts +102 -0
- package/src/combinators/map.ts +106 -0
- package/src/combinators/node.ts +345 -0
- package/src/combinators/not.ts +55 -0
- package/src/combinators/parser.ts +266 -0
- package/src/combinators/peek.ts +63 -0
- package/src/combinators/probe.ts +25 -0
- package/src/combinators/ref.ts +109 -0
- package/src/combinators/regex.ts +170 -0
- package/src/combinators/repeat.ts +561 -0
- package/src/combinators/scanTo.ts +393 -0
- package/src/combinators/sequence.ts +237 -0
- package/src/combinators/token.ts +120 -0
- package/src/combinators/trivia-boundary.ts +48 -0
- package/src/combinators/trivia-skip.ts +852 -0
- package/src/combinators/withCtx.ts +45 -0
- package/src/compiler/build-arity.ts +330 -0
- package/src/compiler/compile-linkable-table.ts +209 -0
- package/src/compiler/degradation.ts +266 -0
- package/src/compiler/fields.ts +191 -0
- package/src/compiler/grammar-coverage-ids.ts +129 -0
- package/src/compiler/ir-serialize.ts +615 -0
- package/src/compiler/linker.ts +954 -0
- package/src/compiler/token-alphabet.ts +209 -0
- package/src/compiler/token-dispatch.ts +353 -0
- package/src/compiler/token-scanner.ts +318 -0
- package/src/compiler/value-usage.ts +147 -0
- package/src/coverage.ts +383 -0
- package/src/cst/capture-buffer.ts +297 -0
- package/src/cst/host-mode.ts +69 -0
- package/src/cst/reflection-attach.ts +11 -0
- package/src/cst/reflection-symbols.ts +9 -0
- package/src/cst/reflection.ts +149 -0
- package/src/cst/relative-spans.ts +207 -0
- package/src/cst/root-trivia-scope.ts +30 -0
- package/src/cst/trivia-charscan.ts +312 -0
- package/src/cst/trivia-entries.ts +285 -0
- package/src/cst/trivia-index.ts +131 -0
- package/src/cst/trivia-kinds.ts +358 -0
- package/src/cst/types.ts +76 -0
- package/src/cst/walk.ts +101 -0
- package/src/functional/doc.ts +959 -0
- package/src/functional/run-tabled.ts +153 -0
- package/src/functional/run.ts +558 -0
- package/src/grammar-metadata.ts +2 -0
- package/src/index.ts +189 -0
- package/src/language-service/index.ts +221 -0
- package/src/line-index.ts +131 -0
- package/src/oracle/digest.ts +620 -0
- package/src/oracle/index.ts +55 -0
- package/src/parse-context.ts +91 -0
- package/src/plugin/direct-builder-static.ts +102 -0
- package/src/plugin/evaluator.ts +1505 -0
- package/src/plugin/free-identifiers.ts +338 -0
- package/src/plugin/index.ts +2281 -0
- package/src/plugin/reducer-resolver.ts +606 -0
- package/src/recovery/scan.ts +155 -0
- package/src/regex/classes.ts +183 -0
- package/src/regex/first-set.ts +316 -0
- package/src/run/index.ts +31 -0
- package/src/spec/ebnf.ts +128 -0
- package/src/spec/index.ts +54 -0
- package/src/spec/model.ts +636 -0
- package/src/spec/railroad-lib.ts +559 -0
- package/src/spec/railroad.ts +256 -0
- package/src/table/assemble.ts +3006 -0
- package/src/table/cell.ts +42 -0
- package/src/table/child-slots.ts +113 -0
- package/src/table/compile-rule-map.ts +336 -0
- package/src/table/compile.ts +244 -0
- package/src/table/duplication-hook.ts +89 -0
- package/src/table/emit-assembly.ts +1777 -0
- package/src/table/emit.ts +429 -0
- package/src/table/encode-baseline.ts +340 -0
- package/src/table/encode.ts +1569 -0
- package/src/table/exec-baseline.ts +537 -0
- package/src/table/exec.ts +1680 -0
- package/src/table/fold.ts +38 -0
- package/src/table/index.ts +108 -0
- package/src/table/inspect.ts +52 -0
- package/src/table/ops.ts +467 -0
- package/src/table/program.ts +876 -0
- package/src/table/run-support.ts +66 -0
- package/src/table/scan-shapes.ts +1578 -0
- package/src/table/site-labels.ts +228 -0
- package/src/table/stamp.ts +229 -0
- package/src/types.ts +563 -0
- package/src/version.ts +16 -0
- package/dist/compiler/codegen.d.ts +0 -334
- package/dist/compiler/codegen.d.ts.map +0 -1
- package/dist/compiler/inline-build.d.ts +0 -11
- package/dist/compiler/inline-build.d.ts.map +0 -1
- package/dist/compiler/inline-callback.d.ts +0 -21
- package/dist/compiler/inline-callback.d.ts.map +0 -1
- package/dist/compiler/line-index.d.ts +0 -2
- package/dist/compiler/line-index.d.ts.map +0 -1
- package/dist/compiler/module-hoist.d.ts +0 -106
- package/dist/compiler/module-hoist.d.ts.map +0 -1
- package/dist/compiler/scannable-run.d.ts.map +0 -1
- package/dist/compiler/scannable-terminal.d.ts +0 -23
- package/dist/compiler/scannable-terminal.d.ts.map +0 -1
- package/dist/compiler/trivia-fast-path.d.ts +0 -38
- package/dist/compiler/trivia-fast-path.d.ts.map +0 -1
|
@@ -0,0 +1,1008 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CHOICE COST — the shape of an ordered choice, and what that shape costs.
|
|
3
|
+
* ========================================================================
|
|
4
|
+
*
|
|
5
|
+
* READ THIS FIRST: THIS MEASURES THE INTERPRETER, NOT THE SHIPPED PARSER.
|
|
6
|
+
* ----------------------------------------------------------------------
|
|
7
|
+
* The interpreter's `firstMatch` loop (src/combinators/choice.ts:149-165) enters every
|
|
8
|
+
* alternative unconditionally. The EMITTED engine does not: `src/table/emit-assembly.ts`
|
|
9
|
+
* hoists a per-choice candidate mask and emits a per-arm first-CHARACTER guard, so an arm
|
|
10
|
+
* whose first set excludes the character at the current position is never entered.
|
|
11
|
+
* First-set gating is this project's single largest parse lever — 25-48% across all
|
|
12
|
+
* four jess parsers — and a naive interpreted profile is blind to every byte of it.
|
|
13
|
+
*
|
|
14
|
+
* So `profileWastedWork` reports TWO columns and derives neither from the other:
|
|
15
|
+
* `attempts`/`wastedBytes` (interpreted) and `gatedAttempts`/`gatedWastedBytes`
|
|
16
|
+
* (modelling codegen's guard — see `compiledFirstCharGate`). Rankings use the compiled
|
|
17
|
+
* column. The DELTA is itself a result: an arm expensive interpreted and cheap gated is
|
|
18
|
+
* ALREADY SOLVED by codegen, and reordering it is unpaid work.
|
|
19
|
+
*
|
|
20
|
+
* WHAT THE GATE ACTUALLY REMOVES — measured, not assumed.
|
|
21
|
+
* Over jess's four dialect grammars (637 kB of CSS, 212 kB of Less):
|
|
22
|
+
*
|
|
23
|
+
* dialect arms whose GATED BYTES differ arm entries removed by the gate
|
|
24
|
+
* css 0 of 158 331,313 of 435,767 (76%)
|
|
25
|
+
* less 0 of 199 413,737 of 562,852 (74%)
|
|
26
|
+
* scss 0 of 200 617,735 of 777,356 (79%)
|
|
27
|
+
* jess 0 of 158 325,655 of 388,263 (84%)
|
|
28
|
+
*
|
|
29
|
+
* The gate removes ATTEMPTS, not rescanned BYTES — and the byte columns are identical
|
|
30
|
+
* to the byte in all four. That is structural, not a coincidence of these corpora: the
|
|
31
|
+
* guard is derived from the arm's first SET, which over-approximates what the arm can
|
|
32
|
+
* start with, so whenever the guard rejects, the arm's own leading terminal would have
|
|
33
|
+
* rejected at the same position having consumed nothing.
|
|
34
|
+
*
|
|
35
|
+
* The practical consequence, and the reason this warning is at the top: any claim of
|
|
36
|
+
* the form "this arm is entered on every X and fails N% of the time" is a statement
|
|
37
|
+
* about the INTERPRETER and is usually false of the shipped parser. A byte ranking
|
|
38
|
+
* survives the correction; a failure-RATE or attempt-count claim does not. Measured on
|
|
39
|
+
* jess's CSS `Value` choice, arm 0 `Percentage` reads as 25,939 entries at 97% failure
|
|
40
|
+
* interpreted, and 7,455 at 90% gated — the same arm, a fifth as important. Arm 4
|
|
41
|
+
* `IdentBlock` is 100% failure in BOTH columns and carries 27,131 bytes; it is the real
|
|
42
|
+
* finding, and only the gated column makes that legible.
|
|
43
|
+
*
|
|
44
|
+
* KNOWN LIMIT OF THE MODEL. On the compose/linkable path a named-rule arm gets a
|
|
45
|
+
* DEFERRED guard (an `@FS:name@` placeholder) that fuse time rewrites with the winning rule's
|
|
46
|
+
* first set — or with `true`, always-try, if it cannot be resolved. This model always
|
|
47
|
+
* resolves, so for cross-artifact ref arms `gatedAttempts` is a LOWER bound: the
|
|
48
|
+
* shipped artifact may enter such an arm more often than modelled. Where a compiled
|
|
49
|
+
* measurement disagrees with this report on an arm that is a bare rule reference, the
|
|
50
|
+
* compiled measurement is right.
|
|
51
|
+
*
|
|
52
|
+
* A PEG `choice` is ordered: arm `i` is only reached once arms `0..i-1` have been
|
|
53
|
+
* TRIED AND FAILED, and every one of those failures re-scanned input that the
|
|
54
|
+
* winning arm then scans again. So each additional alternative, and each earlier
|
|
55
|
+
* alternative, has a time cost — and that cost is invisible in the grammar source,
|
|
56
|
+
* which is why grammar shape drifts without anything going red.
|
|
57
|
+
*
|
|
58
|
+
* This module makes it visible, in two halves that answer two different questions.
|
|
59
|
+
*
|
|
60
|
+
* STATIC — `analyzeChoiceInventory()`
|
|
61
|
+
* Which choice sites have alternatives that share a leading term, and for each:
|
|
62
|
+
* did the compiler LEFT-FACTOR it, or did it decline, and why? `detectSharedPrefix`
|
|
63
|
+
* (src/combinators/choice.ts) already computes this to decide whether to emit the
|
|
64
|
+
* `sharedPrefix` strategy, but it is ALL-OR-NOTHING (every arm must share) and it
|
|
65
|
+
* returns `null` on the first arm that does not qualify — silently, with no record
|
|
66
|
+
* of how close the site came or which arm blocked it. The inventory reports the
|
|
67
|
+
* declines, with the blocking arm and the reason. That set is the refactor backlog,
|
|
68
|
+
* GENERATED rather than noticed by a human reading grammar source.
|
|
69
|
+
*
|
|
70
|
+
* It is complete, not sampled: every `choice` reachable from the rule map appears,
|
|
71
|
+
* including the ones with nothing to report (`groups: []`).
|
|
72
|
+
*
|
|
73
|
+
* DYNAMIC — `profileWastedWork()`
|
|
74
|
+
* How many BYTES were re-scanned because an alternative was tried and failed,
|
|
75
|
+
* attributed per choice site and per arm, over a real corpus. Ordering pathologies
|
|
76
|
+
* fall out of the ranking for free: an arm that fails 98% of the time while sitting
|
|
77
|
+
* first is exactly the top entry.
|
|
78
|
+
*
|
|
79
|
+
* The static half can only see shape. It cannot know that one declined site is on
|
|
80
|
+
* the hot path of every stylesheet and another is reached twice a year. The dynamic
|
|
81
|
+
* half supplies that weight. Neither is sufficient alone.
|
|
82
|
+
*
|
|
83
|
+
* WHY BYTES, AND WHY THIS IS GATEABLE
|
|
84
|
+
* -----------------------------------
|
|
85
|
+
* The metric is a COUNT of input bytes, not a timing. It is a pure function of
|
|
86
|
+
* (grammar, corpus): the same grammar over the same corpus yields the same number on
|
|
87
|
+
* an idle machine and on a machine at load 10. There is no noise floor to argue about
|
|
88
|
+
* and no rebaseline needed for a hardware change. That is what makes it usable as a
|
|
89
|
+
* deterministic gate rather than a benchmark.
|
|
90
|
+
*
|
|
91
|
+
* INTERPRETED MODE ONLY
|
|
92
|
+
* ---------------------
|
|
93
|
+
* Nothing here is emitted into a compiled parser and nothing here costs codegen a
|
|
94
|
+
* single byte or a single millisecond. The profiler does not add an instrumentation
|
|
95
|
+
* flag to the hot combinators either — `src/combinators/choice.ts` is UNTOUCHED by
|
|
96
|
+
* this file. Instrumentation is installed by temporarily substituting arm slots and
|
|
97
|
+
* terminal `parse` methods on the combinator tree, and removed in a `finally`. When
|
|
98
|
+
* you are not profiling, the cost is exactly zero because the shipping code contains
|
|
99
|
+
* no profiling branch to skip.
|
|
100
|
+
*
|
|
101
|
+
* (For contrast: `run({ profile })` — the node/slot-count profiler — was compiled-path
|
|
102
|
+
* only, was never implemented in the interpreter, and now throws rather than reporting
|
|
103
|
+
* zeros. See src/functional/run.ts. This is a different measurement and a new build,
|
|
104
|
+
* not a port of that one.)
|
|
105
|
+
*
|
|
106
|
+
* WHAT THIS INSTRUMENT CANNOT SEE
|
|
107
|
+
* -------------------------------
|
|
108
|
+
* Stated up front, because a diagnostic whose blind spots are undocumented gets read
|
|
109
|
+
* as complete:
|
|
110
|
+
*
|
|
111
|
+
* 0. It measures the INTERPRETER. See the top of this header — this is the blind spot
|
|
112
|
+
* that reorders results rather than merely shifting them, which is why it is not
|
|
113
|
+
* in this list but above it. The `gated*` columns model it; they do not remove it.
|
|
114
|
+
* 1. Only `firstMatch` and `sharedPrefix` choices are instrumented. Those are the
|
|
115
|
+
* two strategies that try arms in order and can fail one. A `disjoint` choice
|
|
116
|
+
* dispatches on the first character and tries exactly one arm; `greedyClassify`
|
|
117
|
+
* runs one regex; `literalsLongestFirst` captures its sorted arm array at
|
|
118
|
+
* construction, so slot substitution cannot reach it. Their sites still appear
|
|
119
|
+
* in the static inventory, with zero dynamic cost recorded and `instrumented:
|
|
120
|
+
* false` so the zero is not mistaken for a measurement.
|
|
121
|
+
* 2. Backtracking that is not a choice arm — a failed `many` element, a failed
|
|
122
|
+
* `optional`, a rolled-back `attempt` — is not attributed. Those bytes are real
|
|
123
|
+
* but they are not an ORDERING decision, which is what this instrument exists to
|
|
124
|
+
* rank.
|
|
125
|
+
* 3. An arm that SUCCEEDS and is then rejected by the `autoNot` check is recorded as
|
|
126
|
+
* a success, because the rejection happens inside the choice loop after the arm
|
|
127
|
+
* has returned. Its rescan is invisible here.
|
|
128
|
+
* 4. `wastedBytes` is input bytes re-scanned, not CPU. An arm that fails after one
|
|
129
|
+
* byte but allocated on the way is cheap by this metric and not by the clock.
|
|
130
|
+
* This ranks ORDERING, and is deliberately not a profiler.
|
|
131
|
+
*
|
|
132
|
+
* SET EXPECTATIONS ACCORDINGLY, because this has been measured. A jess lane acted
|
|
133
|
+
* on a finding from this report and cut rescanned bytes at one route from 27,678
|
|
134
|
+
* to 8,367 — a 69.8% reduction, exactly as this instrument predicts — for ZERO
|
|
135
|
+
* measurable wall-clock change. A large byte win that produces no time win is the
|
|
136
|
+
* expected outcome, not a failure of the fix: re-scanning is cheap per byte, and
|
|
137
|
+
* what makes a grammar slow is usually allocation and node construction, which
|
|
138
|
+
* this metric cannot see. Use it to find and rank ORDERING defects and to prove a
|
|
139
|
+
* restructure did what it claimed. Do not use it to predict a speedup, and do not
|
|
140
|
+
* let a flat benchmark be read as evidence the finding was wrong.
|
|
141
|
+
* 5. Reach is measured by the furthest position at which a TERMINAL was attempted
|
|
142
|
+
* (`literal`/`regex`/`keywords`/`scanTo`). A combinator that consumes without
|
|
143
|
+
* going through one of those would under-report; none exists today.
|
|
144
|
+
*/
|
|
145
|
+
|
|
146
|
+
import type { Combinator, ParseContext, ParseResult, ParserDef } from '../types.ts'
|
|
147
|
+
import type { ChoiceStrategyTag } from './gating.ts'
|
|
148
|
+
import { childrenOf } from './gating.ts'
|
|
149
|
+
import { run } from '../functional/run.ts'
|
|
150
|
+
import { firstSetOf } from '../combinators/first-set.ts'
|
|
151
|
+
import type { FirstSet } from '../types.ts'
|
|
152
|
+
|
|
153
|
+
// ── where a choice sits ──────────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
/** A combinator's location in the grammar: owning rule plus structural path. */
|
|
156
|
+
export type ChoiceSite = {
|
|
157
|
+
/** Nearest enclosing `_ruleName`, or the rule-map key it was reached under. */
|
|
158
|
+
rule: string
|
|
159
|
+
/** Structural path inside that rule, e.g. `seq[2] › node(AtRule) › choice[0]`. */
|
|
160
|
+
path: string
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** The stable identity of a site. Used as the sort key and the baseline key, so it
|
|
164
|
+
* must be derived only from grammar structure — never from a file path or a run. */
|
|
165
|
+
export const choiceSiteKey = (s: ChoiceSite): string =>
|
|
166
|
+
s.path === '' ? s.rule : `${s.rule} › ${s.path}`
|
|
167
|
+
|
|
168
|
+
// ── static inventory ─────────────────────────────────────────────────────────
|
|
169
|
+
|
|
170
|
+
/** Why one ARM contributed no shareable leading term. */
|
|
171
|
+
export type ArmDeclineReason =
|
|
172
|
+
/** The arm is not a (wrapper-peeled) `sequence` — e.g. a bare ref, a nested
|
|
173
|
+
* `choice`, an `optional`. `leadingTermOfArm` peels only node/grammar/transform/
|
|
174
|
+
* label, because those are the wrappers that consume nothing before the sequence. */
|
|
175
|
+
| 'not-a-sequence'
|
|
176
|
+
/** A `sequence` of one term: there is no residual to factor out from. */
|
|
177
|
+
| 'sequence-shorter-than-2'
|
|
178
|
+
/** The leading term is a case-INSENSITIVE literal. Excluded deliberately: the
|
|
179
|
+
* matched text differs from the literal's own value, so a replayed leaf would
|
|
180
|
+
* carry the wrong string. */
|
|
181
|
+
| 'lead-case-insensitive-literal'
|
|
182
|
+
/** The leading term is not a bare literal/regex — a ref, a node, a label, a
|
|
183
|
+
* transform, another choice. Factoring through it would change the value or
|
|
184
|
+
* capture shape of the arm. */
|
|
185
|
+
| 'lead-not-concrete-terminal'
|
|
186
|
+
|
|
187
|
+
/** Why the SITE as a whole was not left-factored. */
|
|
188
|
+
export type SiteDeclineReason =
|
|
189
|
+
/** First-char dispatch already selects exactly one arm; there is nothing to factor.
|
|
190
|
+
* Not a backlog entry — this is the good state. */
|
|
191
|
+
| 'disjoint-dispatch'
|
|
192
|
+
/** At least one arm carries a runtime gate; per-arm predicates are incompatible
|
|
193
|
+
* with the factoring strategies. */
|
|
194
|
+
| 'gated-arms'
|
|
195
|
+
/** `greedyClassify` or `literalsLongestFirst` matched first. Both are stronger than
|
|
196
|
+
* `sharedPrefix` (neither backtracks), so this is also not a backlog entry. */
|
|
197
|
+
| 'strategy-preempted'
|
|
198
|
+
/** Fewer than two arms. */
|
|
199
|
+
| 'fewer-than-two-arms'
|
|
200
|
+
/** At least one arm produced no shareable leading term. See `armDeclines` for which
|
|
201
|
+
* arm and why. THIS is the backlog: a subset of arms may still share a prefix that
|
|
202
|
+
* the all-or-nothing detector never records. */
|
|
203
|
+
| 'arms-not-factorable'
|
|
204
|
+
/** Every arm produced a leading term, but they are not all the same term. Also the
|
|
205
|
+
* backlog: two arms out of nine sharing `@` is real, repeated work. */
|
|
206
|
+
| 'leads-differ'
|
|
207
|
+
|
|
208
|
+
/** A maximal set of arms at one site sharing an identical concrete leading term. */
|
|
209
|
+
export type PrefixGroup = {
|
|
210
|
+
/** Structural key of the shared term. Injective by construction (JSON-encoded), so
|
|
211
|
+
* two structurally different regexes can never collide onto one group. */
|
|
212
|
+
key: string
|
|
213
|
+
/** Human rendering, e.g. `"@"` or `/[-\w]+/`. */
|
|
214
|
+
render: string
|
|
215
|
+
/** Arm indices, ascending. */
|
|
216
|
+
members: readonly number[]
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export type ChoiceInventoryEntry = {
|
|
220
|
+
site: ChoiceSite
|
|
221
|
+
siteKey: string
|
|
222
|
+
arity: number
|
|
223
|
+
strategy: ChoiceStrategyTag
|
|
224
|
+
disjoint: boolean
|
|
225
|
+
gated: boolean
|
|
226
|
+
/** Every group of >= 2 arms sharing a concrete leading term, ordered by member
|
|
227
|
+
* count descending then key. Empty when no two arms share one. */
|
|
228
|
+
groups: readonly PrefixGroup[]
|
|
229
|
+
/** Per-arm reasons, ascending by arm index. Only arms that produced no key. */
|
|
230
|
+
armDeclines: readonly { arm: number; reason: ArmDeclineReason; detail: string }[]
|
|
231
|
+
factored: boolean
|
|
232
|
+
/** Present iff `factored` is false. */
|
|
233
|
+
declineReason?: SiteDeclineReason
|
|
234
|
+
/** Arms that sit in some group but whose site was NOT factored — the count of
|
|
235
|
+
* alternatives currently re-scanning a prefix a sibling already scanned. */
|
|
236
|
+
unfactoredArms: number
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export type ChoiceInventoryReport = {
|
|
240
|
+
readonly schema: 'parseman.choice-inventory/1'
|
|
241
|
+
rules: number
|
|
242
|
+
/** Distinct `choice` combinator instances reachable from the rule map. */
|
|
243
|
+
choiceSites: number
|
|
244
|
+
factoredSites: number
|
|
245
|
+
/** Sites with >= 1 prefix group that were NOT factored. The backlog. */
|
|
246
|
+
backlogSites: number
|
|
247
|
+
backlogArms: number
|
|
248
|
+
/**
|
|
249
|
+
* Rule-map entries that are unresolvable references — a `g.X` hole that only
|
|
250
|
+
* `compose()` binds. Their subtrees were NOT walked, so this report is partial by
|
|
251
|
+
* exactly this much. Ascending. Empty is the only value that means "complete".
|
|
252
|
+
*/
|
|
253
|
+
unresolvedRoots: readonly string[]
|
|
254
|
+
/** Ascending by `siteKey`. Complete — every reachable choice, including the ones
|
|
255
|
+
* with nothing to report. */
|
|
256
|
+
entries: readonly ChoiceInventoryEntry[]
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// ── dynamic profile ──────────────────────────────────────────────────────────
|
|
260
|
+
|
|
261
|
+
export type WastedWorkArm = {
|
|
262
|
+
siteKey: string
|
|
263
|
+
site: ChoiceSite
|
|
264
|
+
arm: number
|
|
265
|
+
/** Short rendering of the arm's head, so the ranked list reads without opening
|
|
266
|
+
* the grammar. */
|
|
267
|
+
label: string
|
|
268
|
+
/** INTERPRETED: every entry, because the interpreter's firstMatch loop gates nothing. */
|
|
269
|
+
attempts: number
|
|
270
|
+
failures: number
|
|
271
|
+
/** INTERPRETED: input bytes re-scanned by this arm's FAILED attempts, summed over the
|
|
272
|
+
* corpus. For one attempt: (furthest position a terminal was tried) - (entry position). */
|
|
273
|
+
wastedBytes: number
|
|
274
|
+
/** True when codegen emits a first-char guard for this arm, i.e. the shipped parser
|
|
275
|
+
* enters it less often than the interpreter does. */
|
|
276
|
+
firstCharGated: boolean
|
|
277
|
+
/** COMPILED MODEL: entries that survive codegen's first-char guard. This is the
|
|
278
|
+
* number the shipped parser actually pays. Equals `attempts` when `firstCharGated`
|
|
279
|
+
* is false. */
|
|
280
|
+
gatedAttempts: number
|
|
281
|
+
gatedFailures: number
|
|
282
|
+
/** COMPILED MODEL: the ranking column. Bytes re-scanned by failed attempts that
|
|
283
|
+
* codegen's guard would NOT have skipped. */
|
|
284
|
+
gatedWastedBytes: number
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export type WastedWorkSite = {
|
|
288
|
+
siteKey: string
|
|
289
|
+
site: ChoiceSite
|
|
290
|
+
strategy: ChoiceStrategyTag
|
|
291
|
+
arity: number
|
|
292
|
+
/** False when the strategy cannot be instrumented; then `wastedBytes` is 0 because
|
|
293
|
+
* nothing was measured, NOT because nothing was wasted. */
|
|
294
|
+
instrumented: boolean
|
|
295
|
+
attempts: number
|
|
296
|
+
failures: number
|
|
297
|
+
wastedBytes: number
|
|
298
|
+
gatedAttempts: number
|
|
299
|
+
gatedFailures: number
|
|
300
|
+
gatedWastedBytes: number
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export type WastedWorkReport = {
|
|
304
|
+
readonly schema: 'parseman.wasted-work/1'
|
|
305
|
+
corpusFiles: number
|
|
306
|
+
corpusBytes: number
|
|
307
|
+
parsedOk: number
|
|
308
|
+
parsedFailed: number
|
|
309
|
+
/** Sites the profiler installed instrumentation on. */
|
|
310
|
+
instrumentedSites: number
|
|
311
|
+
/** Reachable choice sites whose strategy cannot be instrumented (see the module
|
|
312
|
+
* header, blind spot 1). Reported so a low total is not read as a clean grammar. */
|
|
313
|
+
uninstrumentableSites: number
|
|
314
|
+
/** Rule-map entries that are unresolvable references; their subtrees carry no
|
|
315
|
+
* instrumentation, so the total below is a LOWER BOUND by exactly this much. */
|
|
316
|
+
unresolvedRoots: readonly string[]
|
|
317
|
+
/** INTERPRETED total. Read `totalGatedWastedBytes` for what the shipped parser pays. */
|
|
318
|
+
totalWastedBytes: number
|
|
319
|
+
/** COMPILED MODEL total — the headline number, and the one a gate should use. */
|
|
320
|
+
totalGatedWastedBytes: number
|
|
321
|
+
/** Descending by `gatedWastedBytes`, then `wastedBytes`, then `siteKey`, then `arm`. */
|
|
322
|
+
arms: readonly WastedWorkArm[]
|
|
323
|
+
/**
|
|
324
|
+
* ORDERING INVERSIONS: arms that were attempted at least `inversionMinAttempts`
|
|
325
|
+
* times and failed EVERY time, while a later arm at the same site matched.
|
|
326
|
+
*
|
|
327
|
+
* A second ranking, because bytes alone answer the wrong question for the clearest
|
|
328
|
+
* class of defect. Measured on jess's CSS grammar: `StylesheetAtRule › dispatch[3]`
|
|
329
|
+
* has arm 0 failing 19 of 19 attempts while arm 1 matches — an unambiguous ordering
|
|
330
|
+
* bug — yet it is only the 14th largest site by bytes, because the prelude it
|
|
331
|
+
* re-scans is short. Bytes rank what costs most; this ranks what is most obviously
|
|
332
|
+
* WRONG. Both are in the report and neither is derived from the other.
|
|
333
|
+
*
|
|
334
|
+
* Computed from the GATED columns, so an arm codegen already skips is not reported
|
|
335
|
+
* as an inversion in a parser that never enters it.
|
|
336
|
+
*
|
|
337
|
+
* Descending by `gatedAttempts` — NOT by bytes. An arm can fail 100% of its attempts
|
|
338
|
+
* and re-scan zero bytes, because it fails on its own first terminal; jess's `Value`
|
|
339
|
+
* arm 8 `CustomPropertyValue` does exactly that, 925 times, and is the one ungated arm
|
|
340
|
+
* at that site. Ranked by bytes it is invisible; ranked by attempts it is a finding.
|
|
341
|
+
*/
|
|
342
|
+
inversions: readonly WastedWorkArm[]
|
|
343
|
+
/** Descending by `wastedBytes`, then ascending by `siteKey`. */
|
|
344
|
+
sites: readonly WastedWorkSite[]
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ── grammar walk ─────────────────────────────────────────────────────────────
|
|
348
|
+
|
|
349
|
+
// `childrenOf` is imported from ./gating.ts; this module used to carry a copy.
|
|
350
|
+
|
|
351
|
+
const ruleNameOf = (p: Combinator<unknown>): string | undefined =>
|
|
352
|
+
(p as unknown as { _ruleName?: string })._ruleName
|
|
353
|
+
|
|
354
|
+
/** The path segment contributed by one step down. Names the SLOT, not the child, so
|
|
355
|
+
* a path stays stable when a child is replaced by an equivalent one. */
|
|
356
|
+
function slotLabel(d: ParserDef, index: number): string {
|
|
357
|
+
switch (d.tag) {
|
|
358
|
+
case 'sequence': return `seq[${index}]`
|
|
359
|
+
case 'choice': return `choice[${index}]`
|
|
360
|
+
case 'dispatch': return index === 0 ? 'dispatch.selector' : `dispatch[${index - 1}]`
|
|
361
|
+
case 'sepBy': return index === 0 ? 'sepBy.item' : 'sepBy.sep'
|
|
362
|
+
case 'recover': return index === 0 ? 'recover.parser' : 'recover.sentinel'
|
|
363
|
+
case 'scanTo': return index === 0 ? 'scanTo.sentinel' : `scanTo.skip[${index - 1}]`
|
|
364
|
+
case 'grammar': return index === 0 ? 'grammar' : 'grammar.trivia'
|
|
365
|
+
case 'node': return `node(${d.type ?? 'anonymous'})`
|
|
366
|
+
case 'label': return `label(${d.label})`
|
|
367
|
+
case 'field': return `field(${d.name})`
|
|
368
|
+
default: return d.tag
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
type Walked = { p: Combinator<unknown>; d: ParserDef; site: ChoiceSite }
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* A rule-map VALUE may be a reference rather than a definition.
|
|
376
|
+
*
|
|
377
|
+
* `rules()` hands its factory a proxy that mints a `ref()` placeholder on first touch
|
|
378
|
+
* (src/combinators/parser.ts:145-158), so a rule mentioned before it is defined —
|
|
379
|
+
* or defined in a DIFFERENT composed piece — sits in the map as an undefined `lazy`.
|
|
380
|
+
* Measured on jess's CSS grammar: 124 of 176 rule-map entries are such references.
|
|
381
|
+
*
|
|
382
|
+
* A walker that treats those as leaves therefore sees about a third of the grammar
|
|
383
|
+
* and reports a small, clean-looking number — the exact silent-partial failure this
|
|
384
|
+
* module refuses everywhere else. So a ROOT lazy is followed once, to the definition
|
|
385
|
+
* it names. INTERIOR lazies are still leaves: following those would make every rule's
|
|
386
|
+
* walk cover the whole reachable grammar and turn site paths into nonsense.
|
|
387
|
+
*
|
|
388
|
+
* A root reference that cannot resolve (a cross-piece hole that only `compose()`
|
|
389
|
+
* binds) is returned as unresolved and COUNTED, never skipped quietly.
|
|
390
|
+
*/
|
|
391
|
+
function resolveRoot(p: Combinator<unknown>): { ok: true; p: Combinator<unknown> } | { ok: false } {
|
|
392
|
+
let cur = p
|
|
393
|
+
for (let hop = 0; hop < 8; hop++) {
|
|
394
|
+
const d = cur._def as ParserDef
|
|
395
|
+
if (d.tag !== 'lazy') return { ok: true, p: cur }
|
|
396
|
+
let next: Combinator<unknown>
|
|
397
|
+
try { next = d.thunk() } catch { return { ok: false } }
|
|
398
|
+
if (next === cur) return { ok: false }
|
|
399
|
+
cur = next
|
|
400
|
+
}
|
|
401
|
+
return { ok: false }
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* One entry per distinct combinator INSTANCE, in deterministic pre-order.
|
|
406
|
+
*
|
|
407
|
+
* Rule-map order is the caller's array order, and within a rule the walk is the
|
|
408
|
+
* declared child order — no `Map`/`Set` iteration and no filesystem order anywhere,
|
|
409
|
+
* so the same rule map always produces the same sequence of sites.
|
|
410
|
+
*
|
|
411
|
+
* `assertAnalysable` first: the value `compose()`/`composeLeaf()` returns is a fused
|
|
412
|
+
* artifact of parse FUNCTIONS with no `_def` to walk. Handed one of those, a walker
|
|
413
|
+
* that shrugged would report zero sites — a tool's inability wearing the costume of
|
|
414
|
+
* a clean result. That is the one outcome this module never produces.
|
|
415
|
+
*/
|
|
416
|
+
function walkRules(ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>): { walked: Walked[]; unresolvedRoots: string[] } {
|
|
417
|
+
assertAnalysable(ruleMap)
|
|
418
|
+
const out: Walked[] = []
|
|
419
|
+
const seen = new Set<Combinator<unknown>>()
|
|
420
|
+
const unresolvedRoots: string[] = []
|
|
421
|
+
|
|
422
|
+
const visit = (p: Combinator<unknown>, rule: string, path: string): void => {
|
|
423
|
+
if (seen.has(p)) return
|
|
424
|
+
seen.add(p)
|
|
425
|
+
const d = p._def as ParserDef
|
|
426
|
+
const own = ruleNameOf(p)
|
|
427
|
+
const here: ChoiceSite = own !== undefined && own !== rule ? { rule: own, path: '' } : { rule, path }
|
|
428
|
+
out.push({ p, d, site: here })
|
|
429
|
+
const kids = childrenOf(d)
|
|
430
|
+
for (let i = 0; i < kids.length; i++) {
|
|
431
|
+
const seg = slotLabel(d, i)
|
|
432
|
+
visit(kids[i]!, here.rule, here.path === '' ? seg : `${here.path} › ${seg}`)
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
for (const [name, rule] of ruleMap) {
|
|
437
|
+
const r = resolveRoot(rule)
|
|
438
|
+
if (!r.ok) { unresolvedRoots.push(name); continue }
|
|
439
|
+
visit(r.p, name, '')
|
|
440
|
+
}
|
|
441
|
+
unresolvedRoots.sort()
|
|
442
|
+
return { walked: out, unresolvedRoots }
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function assertAnalysable(ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>): void {
|
|
446
|
+
if (ruleMap.length === 0) {
|
|
447
|
+
throw new TypeError(
|
|
448
|
+
'choice-cost: the rule map is EMPTY. Analysing nothing and reporting no findings is '
|
|
449
|
+
+ 'indistinguishable from analysing a clean grammar, so this is refused rather than '
|
|
450
|
+
+ 'answered. Pass the rules() map (or the map given to compileRuleMap/compileLinkable).',
|
|
451
|
+
)
|
|
452
|
+
}
|
|
453
|
+
for (const [name, rule] of ruleMap) {
|
|
454
|
+
const def = (rule as { _def?: unknown } | null | undefined)?._def
|
|
455
|
+
if (def !== null && typeof def === 'object' && typeof (def as { tag?: unknown }).tag === 'string') continue
|
|
456
|
+
throw new TypeError(
|
|
457
|
+
`choice-cost: rule '${name}' is not a combinator (no _def). This analysis walks the `
|
|
458
|
+
+ 'COMBINATOR TREE; the value returned by compose()/composeLeaf() is a fused artifact whose '
|
|
459
|
+
+ 'entries are parse functions. Pass the rules() map itself instead.',
|
|
460
|
+
)
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// ── leading-term analysis (mirrors src/combinators/choice.ts) ─────────────────
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Peel the wrappers that consume nothing before their inner sequence, then return
|
|
468
|
+
* that sequence's FIRST term. Mirrors `leadingTermOfArm` in choice.ts, but reports
|
|
469
|
+
* WHY it gave up instead of returning `null` — the reason is the whole point of the
|
|
470
|
+
* inventory. Kept structurally parallel to the original on purpose: if the two ever
|
|
471
|
+
* disagree the inventory would describe a grammar the compiler does not see.
|
|
472
|
+
*/
|
|
473
|
+
function leadingTermOfArm(arm: Combinator<unknown>): { term: Combinator<unknown> } | { reason: ArmDeclineReason; detail: string } {
|
|
474
|
+
let d = arm._def as ParserDef
|
|
475
|
+
for (;;) {
|
|
476
|
+
if (d.tag === 'node' || d.tag === 'grammar' || d.tag === 'transform' || d.tag === 'label') {
|
|
477
|
+
d = ((d as unknown as { parser: Combinator<unknown> }).parser)._def as ParserDef
|
|
478
|
+
continue
|
|
479
|
+
}
|
|
480
|
+
break
|
|
481
|
+
}
|
|
482
|
+
if (d.tag !== 'sequence') {
|
|
483
|
+
return {
|
|
484
|
+
reason: 'not-a-sequence',
|
|
485
|
+
detail: `arm is \`${d.tag}\`; only node/grammar/transform/label wrappers are peeled, `
|
|
486
|
+
+ 'because they are the ones that consume nothing before the sequence',
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
if (d.parsers.length < 2) {
|
|
490
|
+
return { reason: 'sequence-shorter-than-2', detail: `sequence has ${d.parsers.length} term(s); nothing would remain after factoring` }
|
|
491
|
+
}
|
|
492
|
+
return { term: d.parsers[0]! }
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** Structural key of a bare leading terminal, or the reason it is not shareable.
|
|
496
|
+
* JSON-encoded rather than delimiter-joined: a collision would judge two different
|
|
497
|
+
* regexes identical, which is a correctness bug in a generated parser, not a missed
|
|
498
|
+
* optimisation. */
|
|
499
|
+
function leadingTermKey(term: Combinator<unknown>): { key: string; render: string } | { reason: ArmDeclineReason; detail: string } {
|
|
500
|
+
const d = term._def as ParserDef
|
|
501
|
+
if (d.tag === 'literal') {
|
|
502
|
+
if (d.caseInsensitive) {
|
|
503
|
+
return { reason: 'lead-case-insensitive-literal', detail: `literal ${JSON.stringify(d.value)} is case-insensitive; the matched text can differ from the literal's own value` }
|
|
504
|
+
}
|
|
505
|
+
return { key: `L:${JSON.stringify(d.value)}`, render: JSON.stringify(d.value) }
|
|
506
|
+
}
|
|
507
|
+
if (d.tag === 'regex') {
|
|
508
|
+
return { key: `R:${JSON.stringify([d.source, d.flags])}`, render: `/${d.source}/${d.flags.replace(/[gy]/g, '')}` }
|
|
509
|
+
}
|
|
510
|
+
return {
|
|
511
|
+
reason: 'lead-not-concrete-terminal',
|
|
512
|
+
detail: `leading term is \`${d.tag}\`${d.tag === 'lazy' ? ` (ref${ruleNameOf(term) === undefined ? '' : ` to ${ruleNameOf(term)!}`})` : ''}; factoring through it would change the arm's value or capture shape`,
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** A short, deterministic rendering of an arm's head, for the ranked list. */
|
|
517
|
+
export function armLabel(arm: Combinator<unknown>): string {
|
|
518
|
+
const name = ruleNameOf(arm)
|
|
519
|
+
if (name !== undefined) return name
|
|
520
|
+
let d = arm._def as ParserDef
|
|
521
|
+
for (let hop = 0; hop < 8; hop++) {
|
|
522
|
+
if (d.tag === 'node' && d.type !== undefined) return `node(${d.type})`
|
|
523
|
+
if (d.tag === 'label') return `label(${d.label})`
|
|
524
|
+
if (d.tag === 'field') return `field(${d.name})`
|
|
525
|
+
if (d.tag === 'literal') return JSON.stringify(d.value)
|
|
526
|
+
if (d.tag === 'regex') return `/${d.source}/`
|
|
527
|
+
if (d.tag === 'keywords') return `keywords(${d.words.length})`
|
|
528
|
+
if (d.tag === 'sequence' || d.tag === 'choice') {
|
|
529
|
+
const first = d.parsers[0]
|
|
530
|
+
if (first === undefined) return d.tag
|
|
531
|
+
const inner = ruleNameOf(first)
|
|
532
|
+
if (inner !== undefined) return `${d.tag}(${inner}, …)`
|
|
533
|
+
d = first._def as ParserDef
|
|
534
|
+
continue
|
|
535
|
+
}
|
|
536
|
+
const rec = d as unknown as { parser?: Combinator<unknown> }
|
|
537
|
+
if (rec.parser === undefined) return d.tag
|
|
538
|
+
d = rec.parser._def as ParserDef
|
|
539
|
+
}
|
|
540
|
+
return (arm._def as ParserDef).tag
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// ── HALF 1: static inventory ─────────────────────────────────────────────────
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Every `choice` reachable from `ruleMap`, with its shared-prefix groups and whether
|
|
547
|
+
* the compiler factored the site.
|
|
548
|
+
*
|
|
549
|
+
* Complete by construction: the walk visits every reachable combinator instance and
|
|
550
|
+
* emits an entry for each `choice`, whether or not it has anything to report. A site
|
|
551
|
+
* missing from this report is a site the walk could not reach, never a site that was
|
|
552
|
+
* judged uninteresting.
|
|
553
|
+
*/
|
|
554
|
+
export function analyzeChoiceInventory(
|
|
555
|
+
ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>,
|
|
556
|
+
): ChoiceInventoryReport {
|
|
557
|
+
const { walked, unresolvedRoots } = walkRules(ruleMap)
|
|
558
|
+
const entries: ChoiceInventoryEntry[] = []
|
|
559
|
+
|
|
560
|
+
for (const w of walked) {
|
|
561
|
+
if (w.d.tag !== 'choice') continue
|
|
562
|
+
const d = w.d
|
|
563
|
+
const arity = d.parsers.length
|
|
564
|
+
const strategy = d.strategy.tag as ChoiceStrategyTag
|
|
565
|
+
const gated = d.gates.some(g => g !== null)
|
|
566
|
+
|
|
567
|
+
// Group arms by their leading term. The compiler's detector returns null on the
|
|
568
|
+
// FIRST arm that does not qualify; this keeps going, so a partial group — the
|
|
569
|
+
// case the detector cannot represent at all — is recorded instead of lost.
|
|
570
|
+
const byKey = new Map<string, { render: string; members: number[] }>()
|
|
571
|
+
const armDeclines: { arm: number; reason: ArmDeclineReason; detail: string }[] = []
|
|
572
|
+
for (let i = 0; i < arity; i++) {
|
|
573
|
+
const lead = leadingTermOfArm(d.parsers[i]!)
|
|
574
|
+
if ('reason' in lead) { armDeclines.push({ arm: i, reason: lead.reason, detail: lead.detail }); continue }
|
|
575
|
+
const keyed = leadingTermKey(lead.term)
|
|
576
|
+
if ('reason' in keyed) { armDeclines.push({ arm: i, reason: keyed.reason, detail: keyed.detail }); continue }
|
|
577
|
+
const slot = byKey.get(keyed.key)
|
|
578
|
+
if (slot === undefined) byKey.set(keyed.key, { render: keyed.render, members: [i] })
|
|
579
|
+
else slot.members.push(i)
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
const groups: PrefixGroup[] = [...byKey.entries()]
|
|
583
|
+
.filter(([, v]) => v.members.length >= 2)
|
|
584
|
+
.map(([key, v]) => ({ key, render: v.render, members: v.members }))
|
|
585
|
+
// Explicit sort: Map insertion order is walk order, which is deterministic,
|
|
586
|
+
// but relying on it would make a later walk change reorder the report.
|
|
587
|
+
.sort((a, b) => b.members.length - a.members.length || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
|
|
588
|
+
|
|
589
|
+
const factored = strategy === 'sharedPrefix'
|
|
590
|
+
let declineReason: SiteDeclineReason | undefined
|
|
591
|
+
if (!factored) {
|
|
592
|
+
declineReason =
|
|
593
|
+
arity < 2 ? 'fewer-than-two-arms'
|
|
594
|
+
: d.disjoint ? 'disjoint-dispatch'
|
|
595
|
+
: gated ? 'gated-arms'
|
|
596
|
+
: strategy !== 'firstMatch' ? 'strategy-preempted'
|
|
597
|
+
: armDeclines.length > 0 ? 'arms-not-factorable'
|
|
598
|
+
: 'leads-differ'
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
const inGroups = new Set<number>()
|
|
602
|
+
for (const g of groups) for (const m of g.members) inGroups.add(m)
|
|
603
|
+
|
|
604
|
+
entries.push({
|
|
605
|
+
site: w.site,
|
|
606
|
+
siteKey: choiceSiteKey(w.site),
|
|
607
|
+
arity,
|
|
608
|
+
strategy,
|
|
609
|
+
disjoint: d.disjoint,
|
|
610
|
+
gated,
|
|
611
|
+
groups,
|
|
612
|
+
armDeclines,
|
|
613
|
+
factored,
|
|
614
|
+
...(declineReason === undefined ? {} : { declineReason }),
|
|
615
|
+
unfactoredArms: factored ? 0 : inGroups.size,
|
|
616
|
+
})
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
entries.sort((a, b) => (a.siteKey < b.siteKey ? -1 : a.siteKey > b.siteKey ? 1 : 0))
|
|
620
|
+
|
|
621
|
+
// A site whose decline reason is `disjoint-dispatch` or `strategy-preempted` is
|
|
622
|
+
// NOT backlog: first-char dispatch and the two literal strategies already avoid
|
|
623
|
+
// the re-scan. Counting them would inflate the backlog with sites that are
|
|
624
|
+
// already in the good state.
|
|
625
|
+
const backlog = entries.filter(e =>
|
|
626
|
+
!e.factored && e.groups.length > 0
|
|
627
|
+
&& e.declineReason !== 'disjoint-dispatch' && e.declineReason !== 'strategy-preempted')
|
|
628
|
+
|
|
629
|
+
return {
|
|
630
|
+
schema: 'parseman.choice-inventory/1',
|
|
631
|
+
rules: ruleMap.length,
|
|
632
|
+
choiceSites: entries.length,
|
|
633
|
+
factoredSites: entries.filter(e => e.factored).length,
|
|
634
|
+
backlogSites: backlog.length,
|
|
635
|
+
backlogArms: backlog.reduce((n, e) => n + e.unfactoredArms, 0),
|
|
636
|
+
unresolvedRoots,
|
|
637
|
+
entries,
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
// ── modelling the COMPILED first-char gate ───────────────────────────────────
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* THE BLIND SPOT THIS EXISTS TO CLOSE.
|
|
646
|
+
*
|
|
647
|
+
* The interpreter's `firstMatch` loop (src/combinators/choice.ts:149-165) enters every
|
|
648
|
+
* arm unconditionally. The EMITTED engine does NOT: `src/table/emit-assembly.ts`
|
|
649
|
+
* hoists a per-choice candidate mask and emits a per-arm first-CHARACTER guard, so an arm
|
|
650
|
+
* whose first set excludes the character at the current position is never entered at
|
|
651
|
+
* all. First-set gating is the project's single largest parse lever — 25-48% across all
|
|
652
|
+
* four jess parsers — and an instrument that measures the interpreter is blind to every
|
|
653
|
+
* byte of it.
|
|
654
|
+
*
|
|
655
|
+
* Unmodelled, this does not merely inflate a number, it REORDERS the ranking, because
|
|
656
|
+
* the inflation is proportional to how narrow an arm's first set is — exactly the arms
|
|
657
|
+
* codegen handles best. Measured on jess's `Value` choice: interpreted, arm 0
|
|
658
|
+
* `Percentage` is entered ~25,939 times and fails 97%, and reads as the top finding;
|
|
659
|
+
* compiled, it is gated on `+ - . 0-9` and 44% of those entries never happen. There is
|
|
660
|
+
* no gating gap there to close. Arm 4 `IdentBlock` — 3,743 of 3,743 attempts failed,
|
|
661
|
+
* 74% of that site's rescanned bytes — survives the correction and is the real finding.
|
|
662
|
+
*
|
|
663
|
+
* So both columns are reported and neither is derived from the other. The DELTA is
|
|
664
|
+
* itself a result: an arm expensive interpreted and cheap gated is ALREADY SOLVED by
|
|
665
|
+
* codegen, and a grammar author who reorders it is doing unpaid work.
|
|
666
|
+
*
|
|
667
|
+
* FIDELITY. This replicates codegen's own predicate rather than approximating it, and
|
|
668
|
+
* `test/unit/choice-cost.test.ts` compiles real grammars and asserts the model agrees
|
|
669
|
+
* with the emitted guards, arm by arm — because a hand-copy of another module's logic
|
|
670
|
+
* drifts silently, which is the same class of defect as the blind spot itself.
|
|
671
|
+
*
|
|
672
|
+
* Returns the guarding set, or `null` when codegen emits NO guard for this arm (the
|
|
673
|
+
* compiled parser always enters it, exactly as the interpreter does).
|
|
674
|
+
*/
|
|
675
|
+
function compiledFirstCharGate(p: Combinator<unknown>): FirstSet | null {
|
|
676
|
+
const fs = p._meta.firstSet
|
|
677
|
+
// codegen: needsFirstSetGuard(p) === fs.kind !== 'any' && !canMatchEmptyAtStart(p)
|
|
678
|
+
if (fs.kind !== 'any' && !canMatchEmptyAtStart(p)) return fs
|
|
679
|
+
// codegen's deep-recovery branch: a cached `any` that came only from `ref()`s
|
|
680
|
+
// (which cache `any()` at construction) recovers a real guard from the deep,
|
|
681
|
+
// ref-resolving first set. This is also what the compose/linkable path resolves a
|
|
682
|
+
// DEFERRED `/*@FS:name@*/` placeholder to at fuse time.
|
|
683
|
+
if (fs.kind !== 'any' || canMatchEmptyAtStart(p)) return null
|
|
684
|
+
let deep: FirstSet
|
|
685
|
+
try { deep = firstSetOf(p) } catch { return null }
|
|
686
|
+
return deep.kind === 'ranges' ? deep : null
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* A verbatim replica of `canMatchEmptyAtStart`, which lived in the source lowering
|
|
691
|
+
* deleted in `37c57b5`. Nothing shares this definition today: it is the last copy.
|
|
692
|
+
*
|
|
693
|
+
* Deliberately SHALLOW, and that is not an approximation — it is the behaviour. A
|
|
694
|
+
* `node()`-wrapped nullable parser falls to `default: false`, so codegen treats it as
|
|
695
|
+
* non-nullable and DOES emit a guard for it. Using the public `matchesEmpty()` here
|
|
696
|
+
* instead looks more correct and models the wrong thing: it would report those arms as
|
|
697
|
+
* ungated when the shipped parser gates them. Verified against emitted output.
|
|
698
|
+
*/
|
|
699
|
+
function canMatchEmptyAtStart(p: Combinator<unknown>): boolean {
|
|
700
|
+
const d = p._def as ParserDef
|
|
701
|
+
switch (d.tag) {
|
|
702
|
+
case 'regex': return regexMatchesEmpty(d.source, d.flags)
|
|
703
|
+
case 'optional': case 'many': return true
|
|
704
|
+
case 'transform': case 'label': case 'field':
|
|
705
|
+
return canMatchEmptyAtStart((d as unknown as { parser: Combinator<unknown> }).parser)
|
|
706
|
+
case 'literal': return d.value.length === 0
|
|
707
|
+
default: return false
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/** Does the pattern match the empty string? Falls back to `true` (conservatively
|
|
712
|
+
* nullable, hence always-try) when the pattern cannot be compiled — mirroring codegen. */
|
|
713
|
+
function regexMatchesEmpty(source: string, flags: string): boolean {
|
|
714
|
+
try { return new RegExp(`^(?:${source})$`, flags.replace(/[gy]/g, '')).test('') }
|
|
715
|
+
catch { return true }
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
function inSet(code: number, fs: FirstSet): boolean {
|
|
719
|
+
if (fs.kind === 'any') return true
|
|
720
|
+
if (fs.kind === 'empty') return false
|
|
721
|
+
for (const r of fs.ranges) if (code >= r.lo && code <= r.hi) return true
|
|
722
|
+
return false
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/** Exposed so the fidelity test can compare the model against emitted guards. */
|
|
726
|
+
export function modelledFirstCharGate(p: Combinator<unknown>): FirstSet | null {
|
|
727
|
+
return compiledFirstCharGate(p)
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
// ── HALF 2: corpus-driven wasted work ────────────────────────────────────────
|
|
731
|
+
|
|
732
|
+
/** Terminal tags. Reaching input position `q` means a terminal was ATTEMPTED at `q`,
|
|
733
|
+
* so bumping a high-water mark on terminal entry captures an attempt's furthest
|
|
734
|
+
* reach without inspecting any result. */
|
|
735
|
+
const TERMINAL_TAGS = new Set<ParserDef['tag']>(['literal', 'regex', 'keywords', 'scanTo'])
|
|
736
|
+
|
|
737
|
+
/** Strategies whose arms are tried in order through `def.parsers`, which is the same
|
|
738
|
+
* array object the choice's closure reads at call time — so substituting a slot
|
|
739
|
+
* installs instrumentation with no change to choice.ts and no flag on its hot path.
|
|
740
|
+
* `literalsLongestFirst` is excluded because it captures its sorted arm array at
|
|
741
|
+
* CONSTRUCTION time; a substituted slot would never be read, and instrumentation
|
|
742
|
+
* that silently does nothing is worse than none. */
|
|
743
|
+
const INSTRUMENTABLE = new Set<string>(['firstMatch', 'sharedPrefix'])
|
|
744
|
+
|
|
745
|
+
export type WastedWorkCorpusEntry = {
|
|
746
|
+
/** Stable identifier. Use a repo-relative path, never an absolute one — it appears
|
|
747
|
+
* in the report, and an absolute path would make the report machine-dependent. */
|
|
748
|
+
id: string
|
|
749
|
+
text: string
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
export type ProfileWastedWorkOptions = {
|
|
753
|
+
/** The combinator rule map, for site naming and instrumentation. */
|
|
754
|
+
rules: ReadonlyArray<readonly [string, Combinator<unknown>]>
|
|
755
|
+
/** The rule to start each parse at. A key of `rules`, or a combinator. */
|
|
756
|
+
entry: string | Combinator<unknown>
|
|
757
|
+
corpus: ReadonlyArray<WastedWorkCorpusEntry>
|
|
758
|
+
/** Passed through to `run()`. Must not contain anything machine-dependent. */
|
|
759
|
+
runOptions?: Record<string, unknown>
|
|
760
|
+
/**
|
|
761
|
+
* How to parse one corpus entry. Defaults to `run()` in interpreted mode.
|
|
762
|
+
*
|
|
763
|
+
* Overridable because a grammar can need a driver of its own — jess's Less dialect,
|
|
764
|
+
* for instance, requires `state: { source: input }` because its reducers read it —
|
|
765
|
+
* and because a caller who has already built a driver should not have to duplicate
|
|
766
|
+
* it here. It must be a plain interpreted parse; anything compiled defeats the
|
|
767
|
+
* instrumentation silently, since a compiled artifact never reads `def.parsers`.
|
|
768
|
+
*/
|
|
769
|
+
runner?: (entry: Combinator<unknown>, input: string, options: Record<string, unknown>) => { ok: boolean }
|
|
770
|
+
/** Minimum attempts before an always-failing arm counts as an ordering inversion.
|
|
771
|
+
* Default 4 — below that, "failed every time" is a sample size, not a finding. */
|
|
772
|
+
inversionMinAttempts?: number
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
type ArmRecord = {
|
|
776
|
+
site: ChoiceSite
|
|
777
|
+
siteKey: string
|
|
778
|
+
arm: number
|
|
779
|
+
label: string
|
|
780
|
+
gate: FirstSet | null
|
|
781
|
+
attempts: number
|
|
782
|
+
failures: number
|
|
783
|
+
wastedBytes: number
|
|
784
|
+
gatedAttempts: number
|
|
785
|
+
gatedFailures: number
|
|
786
|
+
gatedWastedBytes: number
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Parse `corpus` with `rules` in INTERPRETED mode, counting input bytes re-scanned
|
|
791
|
+
* by failed choice alternatives.
|
|
792
|
+
*
|
|
793
|
+
* Fails closed on every way of measuring nothing: an empty rule map, an unknown entry
|
|
794
|
+
* rule, an empty corpus, an empty corpus file, or a grammar in which no instrumentable
|
|
795
|
+
* choice site exists. Each of those would otherwise produce a report of zero wasted
|
|
796
|
+
* bytes, which reads as a clean grammar.
|
|
797
|
+
*/
|
|
798
|
+
export function profileWastedWork(opts: ProfileWastedWorkOptions): WastedWorkReport {
|
|
799
|
+
const { rules, corpus } = opts
|
|
800
|
+
const { walked, unresolvedRoots } = walkRules(rules)
|
|
801
|
+
|
|
802
|
+
const entryParser = typeof opts.entry === 'string'
|
|
803
|
+
? rules.find(([n]) => n === opts.entry)?.[1]
|
|
804
|
+
: opts.entry
|
|
805
|
+
if (entryParser === undefined) {
|
|
806
|
+
throw new TypeError(
|
|
807
|
+
`choice-cost: entry rule '${String(opts.entry)}' is not in the rule map `
|
|
808
|
+
+ `(have: ${rules.map(([n]) => n).slice(0, 12).join(', ')}${rules.length > 12 ? ', …' : ''}).`,
|
|
809
|
+
)
|
|
810
|
+
}
|
|
811
|
+
if (corpus.length === 0) {
|
|
812
|
+
throw new TypeError('choice-cost: the corpus is EMPTY. A profile over no input measures nothing; refusing to report zero wasted bytes as a result.')
|
|
813
|
+
}
|
|
814
|
+
for (const f of corpus) {
|
|
815
|
+
if (f.text.length === 0) throw new TypeError(`choice-cost: corpus entry '${f.id}' is EMPTY.`)
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
// ── install ────────────────────────────────────────────────────────────────
|
|
819
|
+
// Two substitutions, both reverted in the `finally`:
|
|
820
|
+
// arms — `def.parsers[i]` is replaced by a delegating combinator that carries
|
|
821
|
+
// its own (site, index). Identity by SLOT, not by object, so an arm
|
|
822
|
+
// subtree shared between two sites is attributed to the right one.
|
|
823
|
+
// terminals — `parse` is replaced in place, because a terminal can sit in a
|
|
824
|
+
// single-child slot (`node.parser`) that the wrapper captured as a
|
|
825
|
+
// closure local and would never re-read from the def.
|
|
826
|
+
let highWater = 0
|
|
827
|
+
const armRecords: ArmRecord[] = []
|
|
828
|
+
const siteTotals = new Map<string, WastedWorkSite>()
|
|
829
|
+
const restore: (() => void)[] = []
|
|
830
|
+
// LIFO: terminal `parse` patches are pushed after the slot substitutions, and a
|
|
831
|
+
// terminal that also sits in a substituted slot must be unpatched before the slot is
|
|
832
|
+
// restored, or the original object would be handed back still wearing a patch.
|
|
833
|
+
const undoAll = (): void => { for (let i = restore.length - 1; i >= 0; i--) restore[i]!() }
|
|
834
|
+
|
|
835
|
+
let instrumentedSites = 0
|
|
836
|
+
let uninstrumentableSites = 0
|
|
837
|
+
|
|
838
|
+
// THE INSTALL IS COVERED TOO, not just the run. Both substitutions mutate a shared
|
|
839
|
+
// combinator tree, so anything that throws part-way through — `compiledFirstCharGate`,
|
|
840
|
+
// `armLabel`, the terminal patch loop — would otherwise leave orphaned instrumentation
|
|
841
|
+
// installed for the rest of the process and every later parse would run through it.
|
|
842
|
+
try {
|
|
843
|
+
for (const w of walked) {
|
|
844
|
+
if (w.d.tag !== 'choice') continue
|
|
845
|
+
const siteKey = choiceSiteKey(w.site)
|
|
846
|
+
const strategy = w.d.strategy.tag as ChoiceStrategyTag
|
|
847
|
+
const canInstrument = INSTRUMENTABLE.has(strategy) && !w.d.disjoint
|
|
848
|
+
const rec: WastedWorkSite = {
|
|
849
|
+
siteKey, site: w.site, strategy, arity: w.d.parsers.length,
|
|
850
|
+
instrumented: canInstrument, attempts: 0, failures: 0, wastedBytes: 0,
|
|
851
|
+
gatedAttempts: 0, gatedFailures: 0, gatedWastedBytes: 0,
|
|
852
|
+
}
|
|
853
|
+
// Two distinct choice INSTANCES can walk to the same site key only if the same
|
|
854
|
+
// slot were visited twice, which `seen` prevents. Keep the first and count the
|
|
855
|
+
// second as uninstrumentable rather than silently merging their numbers.
|
|
856
|
+
if (siteTotals.has(siteKey)) { uninstrumentableSites++; continue }
|
|
857
|
+
siteTotals.set(siteKey, rec)
|
|
858
|
+
if (!canInstrument) { uninstrumentableSites++; continue }
|
|
859
|
+
instrumentedSites++
|
|
860
|
+
|
|
861
|
+
const slots = w.d.parsers
|
|
862
|
+
for (let i = 0; i < slots.length; i++) {
|
|
863
|
+
const original = slots[i]!
|
|
864
|
+
// Computed ONCE per arm, outside the parse loop: the guard is a property of the
|
|
865
|
+
// grammar, not of a position, so recomputing it per attempt would be the profiler
|
|
866
|
+
// paying the cost it is measuring.
|
|
867
|
+
const gate = compiledFirstCharGate(original)
|
|
868
|
+
const armRec: ArmRecord = {
|
|
869
|
+
site: w.site, siteKey, arm: i, label: armLabel(original), gate,
|
|
870
|
+
attempts: 0, failures: 0, wastedBytes: 0,
|
|
871
|
+
gatedAttempts: 0, gatedFailures: 0, gatedWastedBytes: 0,
|
|
872
|
+
}
|
|
873
|
+
armRecords.push(armRec)
|
|
874
|
+
const wrapper: Combinator<unknown> = {
|
|
875
|
+
_tag: original._tag,
|
|
876
|
+
_meta: original._meta,
|
|
877
|
+
_def: original._def,
|
|
878
|
+
parse(input: string, pos: number, ctx: ParseContext): ParseResult<unknown> {
|
|
879
|
+
// Would COMPILED output have entered this arm at all? The arm is still run
|
|
880
|
+
// either way — skipping it would change what the grammar parses, and this is
|
|
881
|
+
// a measurement, not a mode. Only the accounting differs.
|
|
882
|
+
const code = pos < input.length ? (input.codePointAt(pos) ?? -1) : -1
|
|
883
|
+
const entered = gate === null || (code >= 0 && inSet(code, gate))
|
|
884
|
+
|
|
885
|
+
const saved = highWater
|
|
886
|
+
highWater = pos
|
|
887
|
+
const r = original.parse(input, pos, ctx)
|
|
888
|
+
const reach = highWater
|
|
889
|
+
highWater = saved > reach ? saved : reach
|
|
890
|
+
|
|
891
|
+
armRec.attempts++
|
|
892
|
+
rec.attempts++
|
|
893
|
+
if (entered) { armRec.gatedAttempts++; rec.gatedAttempts++ }
|
|
894
|
+
if (!r.ok) {
|
|
895
|
+
armRec.failures++
|
|
896
|
+
rec.failures++
|
|
897
|
+
if (entered) { armRec.gatedFailures++; rec.gatedFailures++ }
|
|
898
|
+
const wasted = reach - pos
|
|
899
|
+
if (wasted > 0) {
|
|
900
|
+
armRec.wastedBytes += wasted
|
|
901
|
+
rec.wastedBytes += wasted
|
|
902
|
+
if (entered) { armRec.gatedWastedBytes += wasted; rec.gatedWastedBytes += wasted }
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
return r
|
|
906
|
+
},
|
|
907
|
+
} as Combinator<unknown>
|
|
908
|
+
// `_ruleName` is read by site naming and by `deriveExpected`; carry it across so
|
|
909
|
+
// the substituted slot is indistinguishable from the original to everything but
|
|
910
|
+
// the counter.
|
|
911
|
+
const rn = ruleNameOf(original)
|
|
912
|
+
if (rn !== undefined) (wrapper as unknown as { _ruleName?: string })._ruleName = rn
|
|
913
|
+
slots[i] = wrapper
|
|
914
|
+
restore.push(() => { slots[i] = original })
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
if (instrumentedSites === 0) {
|
|
919
|
+
// No explicit undo here — the `catch` below owns it, in LIFO order.
|
|
920
|
+
throw new TypeError(
|
|
921
|
+
'choice-cost: the grammar contains NO instrumentable choice site (no `firstMatch` or '
|
|
922
|
+
+ '`sharedPrefix` choice). A wasted-work profile over it would report zero for a reason '
|
|
923
|
+
+ 'that has nothing to do with the grammar being efficient, so it is refused.',
|
|
924
|
+
)
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
for (const w of walked) {
|
|
928
|
+
if (!TERMINAL_TAGS.has(w.d.tag)) continue
|
|
929
|
+
const p = w.p as { parse: Combinator<unknown>['parse'] }
|
|
930
|
+
const original = p.parse
|
|
931
|
+
p.parse = ((input: string, pos: number, ctx: ParseContext) => {
|
|
932
|
+
if (pos > highWater) highWater = pos
|
|
933
|
+
return original.call(w.p as never, input, pos, ctx)
|
|
934
|
+
}) as Combinator<unknown>['parse']
|
|
935
|
+
restore.push(() => { p.parse = original })
|
|
936
|
+
}
|
|
937
|
+
} catch (e) {
|
|
938
|
+
undoAll()
|
|
939
|
+
throw e
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
// ── run ────────────────────────────────────────────────────────────────────
|
|
943
|
+
let parsedOk = 0
|
|
944
|
+
let parsedFailed = 0
|
|
945
|
+
let corpusBytes = 0
|
|
946
|
+
try {
|
|
947
|
+
const runner = opts.runner ?? ((e, input, options) => run(e, input, options))
|
|
948
|
+
for (const f of corpus) {
|
|
949
|
+
corpusBytes += f.text.length
|
|
950
|
+
highWater = 0
|
|
951
|
+
const r = runner(entryParser, f.text, opts.runOptions ?? {})
|
|
952
|
+
if (r.ok) parsedOk++
|
|
953
|
+
else parsedFailed++
|
|
954
|
+
}
|
|
955
|
+
} finally {
|
|
956
|
+
undoAll()
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
const arms: WastedWorkArm[] = armRecords
|
|
960
|
+
.filter(a => a.attempts > 0)
|
|
961
|
+
.map(a => ({
|
|
962
|
+
siteKey: a.siteKey, site: a.site, arm: a.arm, label: a.label,
|
|
963
|
+
attempts: a.attempts, failures: a.failures, wastedBytes: a.wastedBytes,
|
|
964
|
+
firstCharGated: a.gate !== null,
|
|
965
|
+
gatedAttempts: a.gatedAttempts, gatedFailures: a.gatedFailures, gatedWastedBytes: a.gatedWastedBytes,
|
|
966
|
+
}))
|
|
967
|
+
// Ranked by the COMPILED column: what the shipped parser pays is the question a
|
|
968
|
+
// grammar author is actually asking. Interpreted bytes break the tie so the
|
|
969
|
+
// ordering stays total and the report stays diffable.
|
|
970
|
+
.sort((x, y) => y.gatedWastedBytes - x.gatedWastedBytes
|
|
971
|
+
|| y.wastedBytes - x.wastedBytes
|
|
972
|
+
|| (x.siteKey < y.siteKey ? -1 : x.siteKey > y.siteKey ? 1 : 0)
|
|
973
|
+
|| x.arm - y.arm)
|
|
974
|
+
|
|
975
|
+
const sites = [...siteTotals.values()]
|
|
976
|
+
.sort((x, y) => y.gatedWastedBytes - x.gatedWastedBytes
|
|
977
|
+
|| y.wastedBytes - x.wastedBytes
|
|
978
|
+
|| (x.siteKey < y.siteKey ? -1 : x.siteKey > y.siteKey ? 1 : 0))
|
|
979
|
+
|
|
980
|
+
// "A later arm matched" is what separates an inversion from an arm that is simply
|
|
981
|
+
// never the right answer for this corpus. Without it, the last arm of every choice —
|
|
982
|
+
// which fails whenever the whole choice fails — would head the list.
|
|
983
|
+
const minAttempts = opts.inversionMinAttempts ?? 4
|
|
984
|
+
const inversions = arms
|
|
985
|
+
.filter(a =>
|
|
986
|
+
a.gatedAttempts >= minAttempts && a.gatedFailures === a.gatedAttempts
|
|
987
|
+
&& arms.some(b => b.siteKey === a.siteKey && b.arm > a.arm && b.gatedAttempts > b.gatedFailures))
|
|
988
|
+
.sort((x, y) => y.gatedAttempts - x.gatedAttempts
|
|
989
|
+
|| y.gatedWastedBytes - x.gatedWastedBytes
|
|
990
|
+
|| (x.siteKey < y.siteKey ? -1 : x.siteKey > y.siteKey ? 1 : 0)
|
|
991
|
+
|| x.arm - y.arm)
|
|
992
|
+
|
|
993
|
+
return {
|
|
994
|
+
schema: 'parseman.wasted-work/1',
|
|
995
|
+
corpusFiles: corpus.length,
|
|
996
|
+
corpusBytes,
|
|
997
|
+
parsedOk,
|
|
998
|
+
parsedFailed,
|
|
999
|
+
instrumentedSites,
|
|
1000
|
+
uninstrumentableSites,
|
|
1001
|
+
unresolvedRoots,
|
|
1002
|
+
totalWastedBytes: sites.reduce((n, s) => n + s.wastedBytes, 0),
|
|
1003
|
+
totalGatedWastedBytes: sites.reduce((n, s) => n + s.gatedWastedBytes, 0),
|
|
1004
|
+
arms,
|
|
1005
|
+
inversions,
|
|
1006
|
+
sites,
|
|
1007
|
+
}
|
|
1008
|
+
}
|