parseman 0.46.0 → 0.47.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (316) hide show
  1. package/README.md +8 -3
  2. package/dist/analysis/choice-cost-render.d.ts +12 -4
  3. package/dist/analysis/choice-cost-render.d.ts.map +1 -1
  4. package/dist/analysis/choice-cost.d.ts +2 -2
  5. package/dist/analysis/choice-cost.d.ts.map +1 -1
  6. package/dist/analysis/commitment.d.ts +131 -0
  7. package/dist/analysis/commitment.d.ts.map +1 -0
  8. package/dist/analysis/diagnostics.cjs +10339 -7787
  9. package/dist/analysis/diagnostics.cjs.map +3 -4
  10. package/dist/analysis/diagnostics.d.ts +2 -2
  11. package/dist/analysis/diagnostics.js +10339 -7787
  12. package/dist/analysis/diagnostics.js.map +3 -4
  13. package/dist/analysis/duplication.d.ts.map +1 -1
  14. package/dist/analysis/fix.d.ts +3 -3
  15. package/dist/analysis/fix.d.ts.map +1 -1
  16. package/dist/analysis/format-number.d.ts +23 -0
  17. package/dist/analysis/format-number.d.ts.map +1 -0
  18. package/dist/analysis/gating.d.ts +39 -2
  19. package/dist/analysis/gating.d.ts.map +1 -1
  20. package/dist/analysis/rebuild.d.ts.map +1 -1
  21. package/dist/analysis/terminal.d.ts +5 -2
  22. package/dist/analysis/terminal.d.ts.map +1 -1
  23. package/dist/cli/index.cjs +10924 -8506
  24. package/dist/cli/index.cjs.map +3 -4
  25. package/dist/cli/index.js +10924 -8506
  26. package/dist/cli/index.js.map +3 -4
  27. package/dist/combinators/adjacency.d.ts +100 -0
  28. package/dist/combinators/adjacency.d.ts.map +1 -0
  29. package/dist/combinators/choice.d.ts.map +1 -1
  30. package/dist/combinators/completions.d.ts.map +1 -1
  31. package/dist/combinators/dispatch.d.ts +8 -1
  32. package/dist/combinators/dispatch.d.ts.map +1 -1
  33. package/dist/combinators/expect.d.ts.map +1 -1
  34. package/dist/combinators/first-set.d.ts.map +1 -1
  35. package/dist/combinators/gate.d.ts.map +1 -1
  36. package/dist/combinators/grammar.d.ts +22 -0
  37. package/dist/combinators/grammar.d.ts.map +1 -1
  38. package/dist/combinators/literal.d.ts +8 -0
  39. package/dist/combinators/literal.d.ts.map +1 -1
  40. package/dist/combinators/map.d.ts +0 -1
  41. package/dist/combinators/map.d.ts.map +1 -1
  42. package/dist/combinators/node.d.ts +13 -3
  43. package/dist/combinators/node.d.ts.map +1 -1
  44. package/dist/combinators/parser.d.ts +1 -1
  45. package/dist/combinators/parser.d.ts.map +1 -1
  46. package/dist/combinators/ref.d.ts.map +1 -1
  47. package/dist/combinators/regex.d.ts +1 -16
  48. package/dist/combinators/regex.d.ts.map +1 -1
  49. package/dist/combinators/repeat.d.ts +30 -2
  50. package/dist/combinators/repeat.d.ts.map +1 -1
  51. package/dist/combinators/scanTo.d.ts +71 -1
  52. package/dist/combinators/scanTo.d.ts.map +1 -1
  53. package/dist/combinators/sequence.d.ts.map +1 -1
  54. package/dist/combinators/token.d.ts.map +1 -1
  55. package/dist/combinators/trivia-boundary.d.ts +26 -0
  56. package/dist/combinators/trivia-boundary.d.ts.map +1 -0
  57. package/dist/combinators/trivia-skip.d.ts +45 -1
  58. package/dist/combinators/trivia-skip.d.ts.map +1 -1
  59. package/dist/combinators/withCtx.d.ts.map +1 -1
  60. package/dist/compiler/compile-linkable-table.d.ts +104 -0
  61. package/dist/compiler/compile-linkable-table.d.ts.map +1 -0
  62. package/dist/compiler/degradation.d.ts +0 -2
  63. package/dist/compiler/degradation.d.ts.map +1 -1
  64. package/dist/compiler/fields.d.ts +13 -0
  65. package/dist/compiler/fields.d.ts.map +1 -1
  66. package/dist/compiler/grammar-coverage-ids.d.ts +5 -5
  67. package/dist/compiler/grammar-coverage-ids.d.ts.map +1 -1
  68. package/dist/compiler/ir-serialize.d.ts.map +1 -1
  69. package/dist/compiler/linker.d.ts +47 -53
  70. package/dist/compiler/linker.d.ts.map +1 -1
  71. package/dist/compiler/token-alphabet.d.ts +81 -0
  72. package/dist/compiler/token-alphabet.d.ts.map +1 -0
  73. package/dist/compiler/token-dispatch.d.ts +95 -0
  74. package/dist/compiler/token-dispatch.d.ts.map +1 -0
  75. package/dist/compiler/token-scanner.d.ts +53 -0
  76. package/dist/compiler/token-scanner.d.ts.map +1 -0
  77. package/dist/compiler/value-usage.d.ts.map +1 -1
  78. package/dist/coverage.d.ts +1 -1
  79. package/dist/coverage.d.ts.map +1 -1
  80. package/dist/cst/capture-buffer.d.ts +35 -0
  81. package/dist/cst/capture-buffer.d.ts.map +1 -1
  82. package/dist/cst/reflection-attach.d.ts +8 -0
  83. package/dist/cst/reflection-attach.d.ts.map +1 -0
  84. package/dist/cst/reflection-symbols.d.ts +10 -0
  85. package/dist/cst/reflection-symbols.d.ts.map +1 -0
  86. package/dist/cst/reflection.d.ts +2 -4
  87. package/dist/cst/reflection.d.ts.map +1 -1
  88. package/dist/cst/root-trivia-scope.d.ts +26 -0
  89. package/dist/cst/root-trivia-scope.d.ts.map +1 -0
  90. package/dist/cst/trivia-charscan.d.ts +18 -0
  91. package/dist/cst/trivia-charscan.d.ts.map +1 -0
  92. package/dist/cst/trivia-kinds.d.ts +19 -0
  93. package/dist/cst/trivia-kinds.d.ts.map +1 -1
  94. package/dist/functional/doc.d.ts.map +1 -1
  95. package/dist/functional/run-tabled.d.ts +12 -0
  96. package/dist/functional/run-tabled.d.ts.map +1 -0
  97. package/dist/functional/run.d.ts +25 -42
  98. package/dist/functional/run.d.ts.map +1 -1
  99. package/dist/grammar-metadata.d.ts +3 -0
  100. package/dist/grammar-metadata.d.ts.map +1 -0
  101. package/dist/index.cjs +9290 -7181
  102. package/dist/index.cjs.map +3 -4
  103. package/dist/index.d.ts +18 -9
  104. package/dist/index.d.ts.map +1 -1
  105. package/dist/index.js +9300 -7191
  106. package/dist/index.js.map +3 -4
  107. package/dist/language-service/index.cjs +313 -121
  108. package/dist/language-service/index.cjs.map +3 -4
  109. package/dist/language-service/index.js +313 -121
  110. package/dist/language-service/index.js.map +3 -4
  111. package/dist/oracle/index.cjs.map +0 -1
  112. package/dist/oracle/index.js.map +0 -1
  113. package/dist/parse-context.d.ts +49 -0
  114. package/dist/parse-context.d.ts.map +1 -0
  115. package/dist/plugin/evaluator.d.ts +40 -1
  116. package/dist/plugin/evaluator.d.ts.map +1 -1
  117. package/dist/plugin/index.cjs +11175 -9534
  118. package/dist/plugin/index.cjs.map +3 -4
  119. package/dist/plugin/index.d.ts.map +1 -1
  120. package/dist/plugin/index.js +11175 -9534
  121. package/dist/plugin/index.js.map +3 -4
  122. package/dist/regex/classes.d.ts +28 -0
  123. package/dist/regex/classes.d.ts.map +1 -1
  124. package/dist/regex/first-set.d.ts +14 -0
  125. package/dist/regex/first-set.d.ts.map +1 -1
  126. package/dist/run/index.cjs +136 -51
  127. package/dist/run/index.cjs.map +2 -3
  128. package/dist/run/index.js +136 -51
  129. package/dist/run/index.js.map +2 -3
  130. package/dist/spec/index.cjs +2934 -1611
  131. package/dist/spec/index.cjs.map +3 -4
  132. package/dist/spec/index.js +2934 -1611
  133. package/dist/spec/index.js.map +3 -4
  134. package/dist/spec/model.d.ts +8 -0
  135. package/dist/spec/model.d.ts.map +1 -1
  136. package/dist/table/assemble.d.ts +307 -0
  137. package/dist/table/assemble.d.ts.map +1 -0
  138. package/dist/table/cell.d.ts +42 -0
  139. package/dist/table/cell.d.ts.map +1 -0
  140. package/dist/table/child-slots.d.ts +30 -0
  141. package/dist/table/child-slots.d.ts.map +1 -0
  142. package/dist/table/compile-rule-map.d.ts +136 -0
  143. package/dist/table/compile-rule-map.d.ts.map +1 -0
  144. package/dist/table/compile.d.ts +56 -0
  145. package/dist/table/compile.d.ts.map +1 -0
  146. package/dist/table/duplication-hook.d.ts +32 -0
  147. package/dist/table/duplication-hook.d.ts.map +1 -0
  148. package/dist/table/emit-assembly.d.ts +139 -0
  149. package/dist/table/emit-assembly.d.ts.map +1 -0
  150. package/dist/table/emit.d.ts +113 -0
  151. package/dist/table/emit.d.ts.map +1 -0
  152. package/dist/table/encode-baseline.d.ts +16 -0
  153. package/dist/table/encode-baseline.d.ts.map +1 -0
  154. package/dist/table/encode.d.ts +71 -0
  155. package/dist/table/encode.d.ts.map +1 -0
  156. package/dist/table/exec-baseline.d.ts +9 -0
  157. package/dist/table/exec-baseline.d.ts.map +1 -0
  158. package/dist/table/exec.d.ts +72 -0
  159. package/dist/table/exec.d.ts.map +1 -0
  160. package/dist/table/fold.d.ts +25 -0
  161. package/dist/table/fold.d.ts.map +1 -0
  162. package/dist/table/index.cjs +11463 -0
  163. package/dist/table/index.cjs.map +6 -0
  164. package/dist/table/index.d.ts +100 -0
  165. package/dist/table/index.d.ts.map +1 -0
  166. package/dist/table/index.js +11440 -0
  167. package/dist/table/index.js.map +6 -0
  168. package/dist/table/inspect.d.ts +17 -0
  169. package/dist/table/inspect.d.ts.map +1 -0
  170. package/dist/table/ops.d.ts +453 -0
  171. package/dist/table/ops.d.ts.map +1 -0
  172. package/dist/table/program.d.ts +496 -0
  173. package/dist/table/program.d.ts.map +1 -0
  174. package/dist/table/run-support.d.ts +38 -0
  175. package/dist/table/run-support.d.ts.map +1 -0
  176. package/dist/{compiler/scannable-run.d.ts → table/scan-shapes.d.ts} +14 -13
  177. package/dist/table/scan-shapes.d.ts.map +1 -0
  178. package/dist/table/site-labels.d.ts +58 -0
  179. package/dist/table/site-labels.d.ts.map +1 -0
  180. package/dist/table/stamp.d.ts +70 -0
  181. package/dist/table/stamp.d.ts.map +1 -0
  182. package/dist/types.d.ts +80 -24
  183. package/dist/types.d.ts.map +1 -1
  184. package/dist/version.d.ts +1 -1
  185. package/package.json +26 -2
  186. package/src/analysis/choice-cost-gate.ts +359 -0
  187. package/src/analysis/choice-cost-render.ts +293 -0
  188. package/src/analysis/choice-cost.ts +1008 -0
  189. package/src/analysis/commitment.ts +557 -0
  190. package/src/analysis/corpus.ts +130 -0
  191. package/src/analysis/diagnose-render.ts +625 -0
  192. package/src/analysis/diagnose.ts +317 -0
  193. package/src/analysis/diagnostics.ts +33 -0
  194. package/src/analysis/duplication.ts +1955 -0
  195. package/src/analysis/fix-render.ts +217 -0
  196. package/src/analysis/fix.ts +669 -0
  197. package/src/analysis/format-number.ts +31 -0
  198. package/src/analysis/gating.ts +792 -0
  199. package/src/analysis/grammar.ts +72 -0
  200. package/src/analysis/rebuild.ts +287 -0
  201. package/src/analysis/terminal.ts +336 -0
  202. package/src/cli/index.ts +413 -0
  203. package/src/combinators/adjacency.ts +171 -0
  204. package/src/combinators/attempt.ts +41 -0
  205. package/src/combinators/case-fold.ts +90 -0
  206. package/src/combinators/choice.ts +466 -0
  207. package/src/combinators/completions.ts +64 -0
  208. package/src/combinators/dispatch.ts +358 -0
  209. package/src/combinators/expect.ts +159 -0
  210. package/src/combinators/first-set.ts +348 -0
  211. package/src/combinators/gate.ts +50 -0
  212. package/src/combinators/grammar.ts +272 -0
  213. package/src/combinators/keywords.ts +145 -0
  214. package/src/combinators/literal.ts +102 -0
  215. package/src/combinators/map.ts +106 -0
  216. package/src/combinators/node.ts +345 -0
  217. package/src/combinators/not.ts +55 -0
  218. package/src/combinators/parser.ts +266 -0
  219. package/src/combinators/peek.ts +63 -0
  220. package/src/combinators/probe.ts +25 -0
  221. package/src/combinators/ref.ts +109 -0
  222. package/src/combinators/regex.ts +170 -0
  223. package/src/combinators/repeat.ts +561 -0
  224. package/src/combinators/scanTo.ts +393 -0
  225. package/src/combinators/sequence.ts +237 -0
  226. package/src/combinators/token.ts +120 -0
  227. package/src/combinators/trivia-boundary.ts +48 -0
  228. package/src/combinators/trivia-skip.ts +852 -0
  229. package/src/combinators/withCtx.ts +45 -0
  230. package/src/compiler/build-arity.ts +330 -0
  231. package/src/compiler/compile-linkable-table.ts +209 -0
  232. package/src/compiler/degradation.ts +266 -0
  233. package/src/compiler/fields.ts +191 -0
  234. package/src/compiler/grammar-coverage-ids.ts +129 -0
  235. package/src/compiler/ir-serialize.ts +615 -0
  236. package/src/compiler/linker.ts +954 -0
  237. package/src/compiler/token-alphabet.ts +209 -0
  238. package/src/compiler/token-dispatch.ts +353 -0
  239. package/src/compiler/token-scanner.ts +318 -0
  240. package/src/compiler/value-usage.ts +147 -0
  241. package/src/coverage.ts +383 -0
  242. package/src/cst/capture-buffer.ts +297 -0
  243. package/src/cst/host-mode.ts +69 -0
  244. package/src/cst/reflection-attach.ts +11 -0
  245. package/src/cst/reflection-symbols.ts +9 -0
  246. package/src/cst/reflection.ts +149 -0
  247. package/src/cst/relative-spans.ts +207 -0
  248. package/src/cst/root-trivia-scope.ts +30 -0
  249. package/src/cst/trivia-charscan.ts +312 -0
  250. package/src/cst/trivia-entries.ts +285 -0
  251. package/src/cst/trivia-index.ts +131 -0
  252. package/src/cst/trivia-kinds.ts +358 -0
  253. package/src/cst/types.ts +76 -0
  254. package/src/cst/walk.ts +101 -0
  255. package/src/functional/doc.ts +959 -0
  256. package/src/functional/run-tabled.ts +153 -0
  257. package/src/functional/run.ts +558 -0
  258. package/src/grammar-metadata.ts +2 -0
  259. package/src/index.ts +189 -0
  260. package/src/language-service/index.ts +221 -0
  261. package/src/line-index.ts +131 -0
  262. package/src/oracle/digest.ts +620 -0
  263. package/src/oracle/index.ts +55 -0
  264. package/src/parse-context.ts +91 -0
  265. package/src/plugin/direct-builder-static.ts +102 -0
  266. package/src/plugin/evaluator.ts +1505 -0
  267. package/src/plugin/free-identifiers.ts +338 -0
  268. package/src/plugin/index.ts +2281 -0
  269. package/src/plugin/reducer-resolver.ts +606 -0
  270. package/src/recovery/scan.ts +155 -0
  271. package/src/regex/classes.ts +183 -0
  272. package/src/regex/first-set.ts +316 -0
  273. package/src/run/index.ts +31 -0
  274. package/src/spec/ebnf.ts +128 -0
  275. package/src/spec/index.ts +54 -0
  276. package/src/spec/model.ts +636 -0
  277. package/src/spec/railroad-lib.ts +559 -0
  278. package/src/spec/railroad.ts +256 -0
  279. package/src/table/assemble.ts +3006 -0
  280. package/src/table/cell.ts +42 -0
  281. package/src/table/child-slots.ts +113 -0
  282. package/src/table/compile-rule-map.ts +336 -0
  283. package/src/table/compile.ts +244 -0
  284. package/src/table/duplication-hook.ts +89 -0
  285. package/src/table/emit-assembly.ts +1777 -0
  286. package/src/table/emit.ts +429 -0
  287. package/src/table/encode-baseline.ts +340 -0
  288. package/src/table/encode.ts +1569 -0
  289. package/src/table/exec-baseline.ts +537 -0
  290. package/src/table/exec.ts +1680 -0
  291. package/src/table/fold.ts +38 -0
  292. package/src/table/index.ts +108 -0
  293. package/src/table/inspect.ts +52 -0
  294. package/src/table/ops.ts +467 -0
  295. package/src/table/program.ts +876 -0
  296. package/src/table/run-support.ts +66 -0
  297. package/src/table/scan-shapes.ts +1578 -0
  298. package/src/table/site-labels.ts +228 -0
  299. package/src/table/stamp.ts +229 -0
  300. package/src/types.ts +563 -0
  301. package/src/version.ts +16 -0
  302. package/dist/compiler/codegen.d.ts +0 -334
  303. package/dist/compiler/codegen.d.ts.map +0 -1
  304. package/dist/compiler/inline-build.d.ts +0 -11
  305. package/dist/compiler/inline-build.d.ts.map +0 -1
  306. package/dist/compiler/inline-callback.d.ts +0 -21
  307. package/dist/compiler/inline-callback.d.ts.map +0 -1
  308. package/dist/compiler/line-index.d.ts +0 -2
  309. package/dist/compiler/line-index.d.ts.map +0 -1
  310. package/dist/compiler/module-hoist.d.ts +0 -106
  311. package/dist/compiler/module-hoist.d.ts.map +0 -1
  312. package/dist/compiler/scannable-run.d.ts.map +0 -1
  313. package/dist/compiler/scannable-terminal.d.ts +0 -23
  314. package/dist/compiler/scannable-terminal.d.ts.map +0 -1
  315. package/dist/compiler/trivia-fast-path.d.ts +0 -38
  316. package/dist/compiler/trivia-fast-path.d.ts.map +0 -1
@@ -0,0 +1,1008 @@
1
+ /**
2
+ * CHOICE COST — the shape of an ordered choice, and what that shape costs.
3
+ * ========================================================================
4
+ *
5
+ * READ THIS FIRST: THIS MEASURES THE INTERPRETER, NOT THE SHIPPED PARSER.
6
+ * ----------------------------------------------------------------------
7
+ * The interpreter's `firstMatch` loop (src/combinators/choice.ts:149-165) enters every
8
+ * alternative unconditionally. The EMITTED engine does not: `src/table/emit-assembly.ts`
9
+ * hoists a per-choice candidate mask and emits a per-arm first-CHARACTER guard, so an arm
10
+ * whose first set excludes the character at the current position is never entered.
11
+ * First-set gating is this project's single largest parse lever — 25-48% across all
12
+ * four jess parsers — and a naive interpreted profile is blind to every byte of it.
13
+ *
14
+ * So `profileWastedWork` reports TWO columns and derives neither from the other:
15
+ * `attempts`/`wastedBytes` (interpreted) and `gatedAttempts`/`gatedWastedBytes`
16
+ * (modelling codegen's guard — see `compiledFirstCharGate`). Rankings use the compiled
17
+ * column. The DELTA is itself a result: an arm expensive interpreted and cheap gated is
18
+ * ALREADY SOLVED by codegen, and reordering it is unpaid work.
19
+ *
20
+ * WHAT THE GATE ACTUALLY REMOVES — measured, not assumed.
21
+ * Over jess's four dialect grammars (637 kB of CSS, 212 kB of Less):
22
+ *
23
+ * dialect arms whose GATED BYTES differ arm entries removed by the gate
24
+ * css 0 of 158 331,313 of 435,767 (76%)
25
+ * less 0 of 199 413,737 of 562,852 (74%)
26
+ * scss 0 of 200 617,735 of 777,356 (79%)
27
+ * jess 0 of 158 325,655 of 388,263 (84%)
28
+ *
29
+ * The gate removes ATTEMPTS, not rescanned BYTES — and the byte columns are identical
30
+ * to the byte in all four. That is structural, not a coincidence of these corpora: the
31
+ * guard is derived from the arm's first SET, which over-approximates what the arm can
32
+ * start with, so whenever the guard rejects, the arm's own leading terminal would have
33
+ * rejected at the same position having consumed nothing.
34
+ *
35
+ * The practical consequence, and the reason this warning is at the top: any claim of
36
+ * the form "this arm is entered on every X and fails N% of the time" is a statement
37
+ * about the INTERPRETER and is usually false of the shipped parser. A byte ranking
38
+ * survives the correction; a failure-RATE or attempt-count claim does not. Measured on
39
+ * jess's CSS `Value` choice, arm 0 `Percentage` reads as 25,939 entries at 97% failure
40
+ * interpreted, and 7,455 at 90% gated — the same arm, a fifth as important. Arm 4
41
+ * `IdentBlock` is 100% failure in BOTH columns and carries 27,131 bytes; it is the real
42
+ * finding, and only the gated column makes that legible.
43
+ *
44
+ * KNOWN LIMIT OF THE MODEL. On the compose/linkable path a named-rule arm gets a
45
+ * DEFERRED guard (an `@FS:name@` placeholder) that fuse time rewrites with the winning rule's
46
+ * first set — or with `true`, always-try, if it cannot be resolved. This model always
47
+ * resolves, so for cross-artifact ref arms `gatedAttempts` is a LOWER bound: the
48
+ * shipped artifact may enter such an arm more often than modelled. Where a compiled
49
+ * measurement disagrees with this report on an arm that is a bare rule reference, the
50
+ * compiled measurement is right.
51
+ *
52
+ * A PEG `choice` is ordered: arm `i` is only reached once arms `0..i-1` have been
53
+ * TRIED AND FAILED, and every one of those failures re-scanned input that the
54
+ * winning arm then scans again. So each additional alternative, and each earlier
55
+ * alternative, has a time cost — and that cost is invisible in the grammar source,
56
+ * which is why grammar shape drifts without anything going red.
57
+ *
58
+ * This module makes it visible, in two halves that answer two different questions.
59
+ *
60
+ * STATIC — `analyzeChoiceInventory()`
61
+ * Which choice sites have alternatives that share a leading term, and for each:
62
+ * did the compiler LEFT-FACTOR it, or did it decline, and why? `detectSharedPrefix`
63
+ * (src/combinators/choice.ts) already computes this to decide whether to emit the
64
+ * `sharedPrefix` strategy, but it is ALL-OR-NOTHING (every arm must share) and it
65
+ * returns `null` on the first arm that does not qualify — silently, with no record
66
+ * of how close the site came or which arm blocked it. The inventory reports the
67
+ * declines, with the blocking arm and the reason. That set is the refactor backlog,
68
+ * GENERATED rather than noticed by a human reading grammar source.
69
+ *
70
+ * It is complete, not sampled: every `choice` reachable from the rule map appears,
71
+ * including the ones with nothing to report (`groups: []`).
72
+ *
73
+ * DYNAMIC — `profileWastedWork()`
74
+ * How many BYTES were re-scanned because an alternative was tried and failed,
75
+ * attributed per choice site and per arm, over a real corpus. Ordering pathologies
76
+ * fall out of the ranking for free: an arm that fails 98% of the time while sitting
77
+ * first is exactly the top entry.
78
+ *
79
+ * The static half can only see shape. It cannot know that one declined site is on
80
+ * the hot path of every stylesheet and another is reached twice a year. The dynamic
81
+ * half supplies that weight. Neither is sufficient alone.
82
+ *
83
+ * WHY BYTES, AND WHY THIS IS GATEABLE
84
+ * -----------------------------------
85
+ * The metric is a COUNT of input bytes, not a timing. It is a pure function of
86
+ * (grammar, corpus): the same grammar over the same corpus yields the same number on
87
+ * an idle machine and on a machine at load 10. There is no noise floor to argue about
88
+ * and no rebaseline needed for a hardware change. That is what makes it usable as a
89
+ * deterministic gate rather than a benchmark.
90
+ *
91
+ * INTERPRETED MODE ONLY
92
+ * ---------------------
93
+ * Nothing here is emitted into a compiled parser and nothing here costs codegen a
94
+ * single byte or a single millisecond. The profiler does not add an instrumentation
95
+ * flag to the hot combinators either — `src/combinators/choice.ts` is UNTOUCHED by
96
+ * this file. Instrumentation is installed by temporarily substituting arm slots and
97
+ * terminal `parse` methods on the combinator tree, and removed in a `finally`. When
98
+ * you are not profiling, the cost is exactly zero because the shipping code contains
99
+ * no profiling branch to skip.
100
+ *
101
+ * (For contrast: `run({ profile })` — the node/slot-count profiler — was compiled-path
102
+ * only, was never implemented in the interpreter, and now throws rather than reporting
103
+ * zeros. See src/functional/run.ts. This is a different measurement and a new build,
104
+ * not a port of that one.)
105
+ *
106
+ * WHAT THIS INSTRUMENT CANNOT SEE
107
+ * -------------------------------
108
+ * Stated up front, because a diagnostic whose blind spots are undocumented gets read
109
+ * as complete:
110
+ *
111
+ * 0. It measures the INTERPRETER. See the top of this header — this is the blind spot
112
+ * that reorders results rather than merely shifting them, which is why it is not
113
+ * in this list but above it. The `gated*` columns model it; they do not remove it.
114
+ * 1. Only `firstMatch` and `sharedPrefix` choices are instrumented. Those are the
115
+ * two strategies that try arms in order and can fail one. A `disjoint` choice
116
+ * dispatches on the first character and tries exactly one arm; `greedyClassify`
117
+ * runs one regex; `literalsLongestFirst` captures its sorted arm array at
118
+ * construction, so slot substitution cannot reach it. Their sites still appear
119
+ * in the static inventory, with zero dynamic cost recorded and `instrumented:
120
+ * false` so the zero is not mistaken for a measurement.
121
+ * 2. Backtracking that is not a choice arm — a failed `many` element, a failed
122
+ * `optional`, a rolled-back `attempt` — is not attributed. Those bytes are real
123
+ * but they are not an ORDERING decision, which is what this instrument exists to
124
+ * rank.
125
+ * 3. An arm that SUCCEEDS and is then rejected by the `autoNot` check is recorded as
126
+ * a success, because the rejection happens inside the choice loop after the arm
127
+ * has returned. Its rescan is invisible here.
128
+ * 4. `wastedBytes` is input bytes re-scanned, not CPU. An arm that fails after one
129
+ * byte but allocated on the way is cheap by this metric and not by the clock.
130
+ * This ranks ORDERING, and is deliberately not a profiler.
131
+ *
132
+ * SET EXPECTATIONS ACCORDINGLY, because this has been measured. A jess lane acted
133
+ * on a finding from this report and cut rescanned bytes at one route from 27,678
134
+ * to 8,367 — a 69.8% reduction, exactly as this instrument predicts — for ZERO
135
+ * measurable wall-clock change. A large byte win that produces no time win is the
136
+ * expected outcome, not a failure of the fix: re-scanning is cheap per byte, and
137
+ * what makes a grammar slow is usually allocation and node construction, which
138
+ * this metric cannot see. Use it to find and rank ORDERING defects and to prove a
139
+ * restructure did what it claimed. Do not use it to predict a speedup, and do not
140
+ * let a flat benchmark be read as evidence the finding was wrong.
141
+ * 5. Reach is measured by the furthest position at which a TERMINAL was attempted
142
+ * (`literal`/`regex`/`keywords`/`scanTo`). A combinator that consumes without
143
+ * going through one of those would under-report; none exists today.
144
+ */
145
+
146
+ import type { Combinator, ParseContext, ParseResult, ParserDef } from '../types.ts'
147
+ import type { ChoiceStrategyTag } from './gating.ts'
148
+ import { childrenOf } from './gating.ts'
149
+ import { run } from '../functional/run.ts'
150
+ import { firstSetOf } from '../combinators/first-set.ts'
151
+ import type { FirstSet } from '../types.ts'
152
+
153
+ // ── where a choice sits ──────────────────────────────────────────────────────
154
+
155
+ /** A combinator's location in the grammar: owning rule plus structural path. */
156
+ export type ChoiceSite = {
157
+ /** Nearest enclosing `_ruleName`, or the rule-map key it was reached under. */
158
+ rule: string
159
+ /** Structural path inside that rule, e.g. `seq[2] › node(AtRule) › choice[0]`. */
160
+ path: string
161
+ }
162
+
163
+ /** The stable identity of a site. Used as the sort key and the baseline key, so it
164
+ * must be derived only from grammar structure — never from a file path or a run. */
165
+ export const choiceSiteKey = (s: ChoiceSite): string =>
166
+ s.path === '' ? s.rule : `${s.rule} › ${s.path}`
167
+
168
+ // ── static inventory ─────────────────────────────────────────────────────────
169
+
170
+ /** Why one ARM contributed no shareable leading term. */
171
+ export type ArmDeclineReason =
172
+ /** The arm is not a (wrapper-peeled) `sequence` — e.g. a bare ref, a nested
173
+ * `choice`, an `optional`. `leadingTermOfArm` peels only node/grammar/transform/
174
+ * label, because those are the wrappers that consume nothing before the sequence. */
175
+ | 'not-a-sequence'
176
+ /** A `sequence` of one term: there is no residual to factor out from. */
177
+ | 'sequence-shorter-than-2'
178
+ /** The leading term is a case-INSENSITIVE literal. Excluded deliberately: the
179
+ * matched text differs from the literal's own value, so a replayed leaf would
180
+ * carry the wrong string. */
181
+ | 'lead-case-insensitive-literal'
182
+ /** The leading term is not a bare literal/regex — a ref, a node, a label, a
183
+ * transform, another choice. Factoring through it would change the value or
184
+ * capture shape of the arm. */
185
+ | 'lead-not-concrete-terminal'
186
+
187
+ /** Why the SITE as a whole was not left-factored. */
188
+ export type SiteDeclineReason =
189
+ /** First-char dispatch already selects exactly one arm; there is nothing to factor.
190
+ * Not a backlog entry — this is the good state. */
191
+ | 'disjoint-dispatch'
192
+ /** At least one arm carries a runtime gate; per-arm predicates are incompatible
193
+ * with the factoring strategies. */
194
+ | 'gated-arms'
195
+ /** `greedyClassify` or `literalsLongestFirst` matched first. Both are stronger than
196
+ * `sharedPrefix` (neither backtracks), so this is also not a backlog entry. */
197
+ | 'strategy-preempted'
198
+ /** Fewer than two arms. */
199
+ | 'fewer-than-two-arms'
200
+ /** At least one arm produced no shareable leading term. See `armDeclines` for which
201
+ * arm and why. THIS is the backlog: a subset of arms may still share a prefix that
202
+ * the all-or-nothing detector never records. */
203
+ | 'arms-not-factorable'
204
+ /** Every arm produced a leading term, but they are not all the same term. Also the
205
+ * backlog: two arms out of nine sharing `@` is real, repeated work. */
206
+ | 'leads-differ'
207
+
208
+ /** A maximal set of arms at one site sharing an identical concrete leading term. */
209
+ export type PrefixGroup = {
210
+ /** Structural key of the shared term. Injective by construction (JSON-encoded), so
211
+ * two structurally different regexes can never collide onto one group. */
212
+ key: string
213
+ /** Human rendering, e.g. `"@"` or `/[-\w]+/`. */
214
+ render: string
215
+ /** Arm indices, ascending. */
216
+ members: readonly number[]
217
+ }
218
+
219
+ export type ChoiceInventoryEntry = {
220
+ site: ChoiceSite
221
+ siteKey: string
222
+ arity: number
223
+ strategy: ChoiceStrategyTag
224
+ disjoint: boolean
225
+ gated: boolean
226
+ /** Every group of >= 2 arms sharing a concrete leading term, ordered by member
227
+ * count descending then key. Empty when no two arms share one. */
228
+ groups: readonly PrefixGroup[]
229
+ /** Per-arm reasons, ascending by arm index. Only arms that produced no key. */
230
+ armDeclines: readonly { arm: number; reason: ArmDeclineReason; detail: string }[]
231
+ factored: boolean
232
+ /** Present iff `factored` is false. */
233
+ declineReason?: SiteDeclineReason
234
+ /** Arms that sit in some group but whose site was NOT factored — the count of
235
+ * alternatives currently re-scanning a prefix a sibling already scanned. */
236
+ unfactoredArms: number
237
+ }
238
+
239
+ export type ChoiceInventoryReport = {
240
+ readonly schema: 'parseman.choice-inventory/1'
241
+ rules: number
242
+ /** Distinct `choice` combinator instances reachable from the rule map. */
243
+ choiceSites: number
244
+ factoredSites: number
245
+ /** Sites with >= 1 prefix group that were NOT factored. The backlog. */
246
+ backlogSites: number
247
+ backlogArms: number
248
+ /**
249
+ * Rule-map entries that are unresolvable references — a `g.X` hole that only
250
+ * `compose()` binds. Their subtrees were NOT walked, so this report is partial by
251
+ * exactly this much. Ascending. Empty is the only value that means "complete".
252
+ */
253
+ unresolvedRoots: readonly string[]
254
+ /** Ascending by `siteKey`. Complete — every reachable choice, including the ones
255
+ * with nothing to report. */
256
+ entries: readonly ChoiceInventoryEntry[]
257
+ }
258
+
259
+ // ── dynamic profile ──────────────────────────────────────────────────────────
260
+
261
+ export type WastedWorkArm = {
262
+ siteKey: string
263
+ site: ChoiceSite
264
+ arm: number
265
+ /** Short rendering of the arm's head, so the ranked list reads without opening
266
+ * the grammar. */
267
+ label: string
268
+ /** INTERPRETED: every entry, because the interpreter's firstMatch loop gates nothing. */
269
+ attempts: number
270
+ failures: number
271
+ /** INTERPRETED: input bytes re-scanned by this arm's FAILED attempts, summed over the
272
+ * corpus. For one attempt: (furthest position a terminal was tried) - (entry position). */
273
+ wastedBytes: number
274
+ /** True when codegen emits a first-char guard for this arm, i.e. the shipped parser
275
+ * enters it less often than the interpreter does. */
276
+ firstCharGated: boolean
277
+ /** COMPILED MODEL: entries that survive codegen's first-char guard. This is the
278
+ * number the shipped parser actually pays. Equals `attempts` when `firstCharGated`
279
+ * is false. */
280
+ gatedAttempts: number
281
+ gatedFailures: number
282
+ /** COMPILED MODEL: the ranking column. Bytes re-scanned by failed attempts that
283
+ * codegen's guard would NOT have skipped. */
284
+ gatedWastedBytes: number
285
+ }
286
+
287
+ export type WastedWorkSite = {
288
+ siteKey: string
289
+ site: ChoiceSite
290
+ strategy: ChoiceStrategyTag
291
+ arity: number
292
+ /** False when the strategy cannot be instrumented; then `wastedBytes` is 0 because
293
+ * nothing was measured, NOT because nothing was wasted. */
294
+ instrumented: boolean
295
+ attempts: number
296
+ failures: number
297
+ wastedBytes: number
298
+ gatedAttempts: number
299
+ gatedFailures: number
300
+ gatedWastedBytes: number
301
+ }
302
+
303
+ export type WastedWorkReport = {
304
+ readonly schema: 'parseman.wasted-work/1'
305
+ corpusFiles: number
306
+ corpusBytes: number
307
+ parsedOk: number
308
+ parsedFailed: number
309
+ /** Sites the profiler installed instrumentation on. */
310
+ instrumentedSites: number
311
+ /** Reachable choice sites whose strategy cannot be instrumented (see the module
312
+ * header, blind spot 1). Reported so a low total is not read as a clean grammar. */
313
+ uninstrumentableSites: number
314
+ /** Rule-map entries that are unresolvable references; their subtrees carry no
315
+ * instrumentation, so the total below is a LOWER BOUND by exactly this much. */
316
+ unresolvedRoots: readonly string[]
317
+ /** INTERPRETED total. Read `totalGatedWastedBytes` for what the shipped parser pays. */
318
+ totalWastedBytes: number
319
+ /** COMPILED MODEL total — the headline number, and the one a gate should use. */
320
+ totalGatedWastedBytes: number
321
+ /** Descending by `gatedWastedBytes`, then `wastedBytes`, then `siteKey`, then `arm`. */
322
+ arms: readonly WastedWorkArm[]
323
+ /**
324
+ * ORDERING INVERSIONS: arms that were attempted at least `inversionMinAttempts`
325
+ * times and failed EVERY time, while a later arm at the same site matched.
326
+ *
327
+ * A second ranking, because bytes alone answer the wrong question for the clearest
328
+ * class of defect. Measured on jess's CSS grammar: `StylesheetAtRule › dispatch[3]`
329
+ * has arm 0 failing 19 of 19 attempts while arm 1 matches — an unambiguous ordering
330
+ * bug — yet it is only the 14th largest site by bytes, because the prelude it
331
+ * re-scans is short. Bytes rank what costs most; this ranks what is most obviously
332
+ * WRONG. Both are in the report and neither is derived from the other.
333
+ *
334
+ * Computed from the GATED columns, so an arm codegen already skips is not reported
335
+ * as an inversion in a parser that never enters it.
336
+ *
337
+ * Descending by `gatedAttempts` — NOT by bytes. An arm can fail 100% of its attempts
338
+ * and re-scan zero bytes, because it fails on its own first terminal; jess's `Value`
339
+ * arm 8 `CustomPropertyValue` does exactly that, 925 times, and is the one ungated arm
340
+ * at that site. Ranked by bytes it is invisible; ranked by attempts it is a finding.
341
+ */
342
+ inversions: readonly WastedWorkArm[]
343
+ /** Descending by `wastedBytes`, then ascending by `siteKey`. */
344
+ sites: readonly WastedWorkSite[]
345
+ }
346
+
347
+ // ── grammar walk ─────────────────────────────────────────────────────────────
348
+
349
+ // `childrenOf` is imported from ./gating.ts; this module used to carry a copy.
350
+
351
+ const ruleNameOf = (p: Combinator<unknown>): string | undefined =>
352
+ (p as unknown as { _ruleName?: string })._ruleName
353
+
354
+ /** The path segment contributed by one step down. Names the SLOT, not the child, so
355
+ * a path stays stable when a child is replaced by an equivalent one. */
356
+ function slotLabel(d: ParserDef, index: number): string {
357
+ switch (d.tag) {
358
+ case 'sequence': return `seq[${index}]`
359
+ case 'choice': return `choice[${index}]`
360
+ case 'dispatch': return index === 0 ? 'dispatch.selector' : `dispatch[${index - 1}]`
361
+ case 'sepBy': return index === 0 ? 'sepBy.item' : 'sepBy.sep'
362
+ case 'recover': return index === 0 ? 'recover.parser' : 'recover.sentinel'
363
+ case 'scanTo': return index === 0 ? 'scanTo.sentinel' : `scanTo.skip[${index - 1}]`
364
+ case 'grammar': return index === 0 ? 'grammar' : 'grammar.trivia'
365
+ case 'node': return `node(${d.type ?? 'anonymous'})`
366
+ case 'label': return `label(${d.label})`
367
+ case 'field': return `field(${d.name})`
368
+ default: return d.tag
369
+ }
370
+ }
371
+
372
+ type Walked = { p: Combinator<unknown>; d: ParserDef; site: ChoiceSite }
373
+
374
+ /**
375
+ * A rule-map VALUE may be a reference rather than a definition.
376
+ *
377
+ * `rules()` hands its factory a proxy that mints a `ref()` placeholder on first touch
378
+ * (src/combinators/parser.ts:145-158), so a rule mentioned before it is defined —
379
+ * or defined in a DIFFERENT composed piece — sits in the map as an undefined `lazy`.
380
+ * Measured on jess's CSS grammar: 124 of 176 rule-map entries are such references.
381
+ *
382
+ * A walker that treats those as leaves therefore sees about a third of the grammar
383
+ * and reports a small, clean-looking number — the exact silent-partial failure this
384
+ * module refuses everywhere else. So a ROOT lazy is followed once, to the definition
385
+ * it names. INTERIOR lazies are still leaves: following those would make every rule's
386
+ * walk cover the whole reachable grammar and turn site paths into nonsense.
387
+ *
388
+ * A root reference that cannot resolve (a cross-piece hole that only `compose()`
389
+ * binds) is returned as unresolved and COUNTED, never skipped quietly.
390
+ */
391
+ function resolveRoot(p: Combinator<unknown>): { ok: true; p: Combinator<unknown> } | { ok: false } {
392
+ let cur = p
393
+ for (let hop = 0; hop < 8; hop++) {
394
+ const d = cur._def as ParserDef
395
+ if (d.tag !== 'lazy') return { ok: true, p: cur }
396
+ let next: Combinator<unknown>
397
+ try { next = d.thunk() } catch { return { ok: false } }
398
+ if (next === cur) return { ok: false }
399
+ cur = next
400
+ }
401
+ return { ok: false }
402
+ }
403
+
404
+ /**
405
+ * One entry per distinct combinator INSTANCE, in deterministic pre-order.
406
+ *
407
+ * Rule-map order is the caller's array order, and within a rule the walk is the
408
+ * declared child order — no `Map`/`Set` iteration and no filesystem order anywhere,
409
+ * so the same rule map always produces the same sequence of sites.
410
+ *
411
+ * `assertAnalysable` first: the value `compose()`/`composeLeaf()` returns is a fused
412
+ * artifact of parse FUNCTIONS with no `_def` to walk. Handed one of those, a walker
413
+ * that shrugged would report zero sites — a tool's inability wearing the costume of
414
+ * a clean result. That is the one outcome this module never produces.
415
+ */
416
+ function walkRules(ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>): { walked: Walked[]; unresolvedRoots: string[] } {
417
+ assertAnalysable(ruleMap)
418
+ const out: Walked[] = []
419
+ const seen = new Set<Combinator<unknown>>()
420
+ const unresolvedRoots: string[] = []
421
+
422
+ const visit = (p: Combinator<unknown>, rule: string, path: string): void => {
423
+ if (seen.has(p)) return
424
+ seen.add(p)
425
+ const d = p._def as ParserDef
426
+ const own = ruleNameOf(p)
427
+ const here: ChoiceSite = own !== undefined && own !== rule ? { rule: own, path: '' } : { rule, path }
428
+ out.push({ p, d, site: here })
429
+ const kids = childrenOf(d)
430
+ for (let i = 0; i < kids.length; i++) {
431
+ const seg = slotLabel(d, i)
432
+ visit(kids[i]!, here.rule, here.path === '' ? seg : `${here.path} › ${seg}`)
433
+ }
434
+ }
435
+
436
+ for (const [name, rule] of ruleMap) {
437
+ const r = resolveRoot(rule)
438
+ if (!r.ok) { unresolvedRoots.push(name); continue }
439
+ visit(r.p, name, '')
440
+ }
441
+ unresolvedRoots.sort()
442
+ return { walked: out, unresolvedRoots }
443
+ }
444
+
445
+ function assertAnalysable(ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>): void {
446
+ if (ruleMap.length === 0) {
447
+ throw new TypeError(
448
+ 'choice-cost: the rule map is EMPTY. Analysing nothing and reporting no findings is '
449
+ + 'indistinguishable from analysing a clean grammar, so this is refused rather than '
450
+ + 'answered. Pass the rules() map (or the map given to compileRuleMap/compileLinkable).',
451
+ )
452
+ }
453
+ for (const [name, rule] of ruleMap) {
454
+ const def = (rule as { _def?: unknown } | null | undefined)?._def
455
+ if (def !== null && typeof def === 'object' && typeof (def as { tag?: unknown }).tag === 'string') continue
456
+ throw new TypeError(
457
+ `choice-cost: rule '${name}' is not a combinator (no _def). This analysis walks the `
458
+ + 'COMBINATOR TREE; the value returned by compose()/composeLeaf() is a fused artifact whose '
459
+ + 'entries are parse functions. Pass the rules() map itself instead.',
460
+ )
461
+ }
462
+ }
463
+
464
+ // ── leading-term analysis (mirrors src/combinators/choice.ts) ─────────────────
465
+
466
+ /**
467
+ * Peel the wrappers that consume nothing before their inner sequence, then return
468
+ * that sequence's FIRST term. Mirrors `leadingTermOfArm` in choice.ts, but reports
469
+ * WHY it gave up instead of returning `null` — the reason is the whole point of the
470
+ * inventory. Kept structurally parallel to the original on purpose: if the two ever
471
+ * disagree the inventory would describe a grammar the compiler does not see.
472
+ */
473
+ function leadingTermOfArm(arm: Combinator<unknown>): { term: Combinator<unknown> } | { reason: ArmDeclineReason; detail: string } {
474
+ let d = arm._def as ParserDef
475
+ for (;;) {
476
+ if (d.tag === 'node' || d.tag === 'grammar' || d.tag === 'transform' || d.tag === 'label') {
477
+ d = ((d as unknown as { parser: Combinator<unknown> }).parser)._def as ParserDef
478
+ continue
479
+ }
480
+ break
481
+ }
482
+ if (d.tag !== 'sequence') {
483
+ return {
484
+ reason: 'not-a-sequence',
485
+ detail: `arm is \`${d.tag}\`; only node/grammar/transform/label wrappers are peeled, `
486
+ + 'because they are the ones that consume nothing before the sequence',
487
+ }
488
+ }
489
+ if (d.parsers.length < 2) {
490
+ return { reason: 'sequence-shorter-than-2', detail: `sequence has ${d.parsers.length} term(s); nothing would remain after factoring` }
491
+ }
492
+ return { term: d.parsers[0]! }
493
+ }
494
+
495
+ /** Structural key of a bare leading terminal, or the reason it is not shareable.
496
+ * JSON-encoded rather than delimiter-joined: a collision would judge two different
497
+ * regexes identical, which is a correctness bug in a generated parser, not a missed
498
+ * optimisation. */
499
+ function leadingTermKey(term: Combinator<unknown>): { key: string; render: string } | { reason: ArmDeclineReason; detail: string } {
500
+ const d = term._def as ParserDef
501
+ if (d.tag === 'literal') {
502
+ if (d.caseInsensitive) {
503
+ return { reason: 'lead-case-insensitive-literal', detail: `literal ${JSON.stringify(d.value)} is case-insensitive; the matched text can differ from the literal's own value` }
504
+ }
505
+ return { key: `L:${JSON.stringify(d.value)}`, render: JSON.stringify(d.value) }
506
+ }
507
+ if (d.tag === 'regex') {
508
+ return { key: `R:${JSON.stringify([d.source, d.flags])}`, render: `/${d.source}/${d.flags.replace(/[gy]/g, '')}` }
509
+ }
510
+ return {
511
+ reason: 'lead-not-concrete-terminal',
512
+ detail: `leading term is \`${d.tag}\`${d.tag === 'lazy' ? ` (ref${ruleNameOf(term) === undefined ? '' : ` to ${ruleNameOf(term)!}`})` : ''}; factoring through it would change the arm's value or capture shape`,
513
+ }
514
+ }
515
+
516
+ /** A short, deterministic rendering of an arm's head, for the ranked list. */
517
+ export function armLabel(arm: Combinator<unknown>): string {
518
+ const name = ruleNameOf(arm)
519
+ if (name !== undefined) return name
520
+ let d = arm._def as ParserDef
521
+ for (let hop = 0; hop < 8; hop++) {
522
+ if (d.tag === 'node' && d.type !== undefined) return `node(${d.type})`
523
+ if (d.tag === 'label') return `label(${d.label})`
524
+ if (d.tag === 'field') return `field(${d.name})`
525
+ if (d.tag === 'literal') return JSON.stringify(d.value)
526
+ if (d.tag === 'regex') return `/${d.source}/`
527
+ if (d.tag === 'keywords') return `keywords(${d.words.length})`
528
+ if (d.tag === 'sequence' || d.tag === 'choice') {
529
+ const first = d.parsers[0]
530
+ if (first === undefined) return d.tag
531
+ const inner = ruleNameOf(first)
532
+ if (inner !== undefined) return `${d.tag}(${inner}, …)`
533
+ d = first._def as ParserDef
534
+ continue
535
+ }
536
+ const rec = d as unknown as { parser?: Combinator<unknown> }
537
+ if (rec.parser === undefined) return d.tag
538
+ d = rec.parser._def as ParserDef
539
+ }
540
+ return (arm._def as ParserDef).tag
541
+ }
542
+
543
+ // ── HALF 1: static inventory ─────────────────────────────────────────────────
544
+
545
+ /**
546
+ * Every `choice` reachable from `ruleMap`, with its shared-prefix groups and whether
547
+ * the compiler factored the site.
548
+ *
549
+ * Complete by construction: the walk visits every reachable combinator instance and
550
+ * emits an entry for each `choice`, whether or not it has anything to report. A site
551
+ * missing from this report is a site the walk could not reach, never a site that was
552
+ * judged uninteresting.
553
+ */
554
+ export function analyzeChoiceInventory(
555
+ ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>,
556
+ ): ChoiceInventoryReport {
557
+ const { walked, unresolvedRoots } = walkRules(ruleMap)
558
+ const entries: ChoiceInventoryEntry[] = []
559
+
560
+ for (const w of walked) {
561
+ if (w.d.tag !== 'choice') continue
562
+ const d = w.d
563
+ const arity = d.parsers.length
564
+ const strategy = d.strategy.tag as ChoiceStrategyTag
565
+ const gated = d.gates.some(g => g !== null)
566
+
567
+ // Group arms by their leading term. The compiler's detector returns null on the
568
+ // FIRST arm that does not qualify; this keeps going, so a partial group — the
569
+ // case the detector cannot represent at all — is recorded instead of lost.
570
+ const byKey = new Map<string, { render: string; members: number[] }>()
571
+ const armDeclines: { arm: number; reason: ArmDeclineReason; detail: string }[] = []
572
+ for (let i = 0; i < arity; i++) {
573
+ const lead = leadingTermOfArm(d.parsers[i]!)
574
+ if ('reason' in lead) { armDeclines.push({ arm: i, reason: lead.reason, detail: lead.detail }); continue }
575
+ const keyed = leadingTermKey(lead.term)
576
+ if ('reason' in keyed) { armDeclines.push({ arm: i, reason: keyed.reason, detail: keyed.detail }); continue }
577
+ const slot = byKey.get(keyed.key)
578
+ if (slot === undefined) byKey.set(keyed.key, { render: keyed.render, members: [i] })
579
+ else slot.members.push(i)
580
+ }
581
+
582
+ const groups: PrefixGroup[] = [...byKey.entries()]
583
+ .filter(([, v]) => v.members.length >= 2)
584
+ .map(([key, v]) => ({ key, render: v.render, members: v.members }))
585
+ // Explicit sort: Map insertion order is walk order, which is deterministic,
586
+ // but relying on it would make a later walk change reorder the report.
587
+ .sort((a, b) => b.members.length - a.members.length || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
588
+
589
+ const factored = strategy === 'sharedPrefix'
590
+ let declineReason: SiteDeclineReason | undefined
591
+ if (!factored) {
592
+ declineReason =
593
+ arity < 2 ? 'fewer-than-two-arms'
594
+ : d.disjoint ? 'disjoint-dispatch'
595
+ : gated ? 'gated-arms'
596
+ : strategy !== 'firstMatch' ? 'strategy-preempted'
597
+ : armDeclines.length > 0 ? 'arms-not-factorable'
598
+ : 'leads-differ'
599
+ }
600
+
601
+ const inGroups = new Set<number>()
602
+ for (const g of groups) for (const m of g.members) inGroups.add(m)
603
+
604
+ entries.push({
605
+ site: w.site,
606
+ siteKey: choiceSiteKey(w.site),
607
+ arity,
608
+ strategy,
609
+ disjoint: d.disjoint,
610
+ gated,
611
+ groups,
612
+ armDeclines,
613
+ factored,
614
+ ...(declineReason === undefined ? {} : { declineReason }),
615
+ unfactoredArms: factored ? 0 : inGroups.size,
616
+ })
617
+ }
618
+
619
+ entries.sort((a, b) => (a.siteKey < b.siteKey ? -1 : a.siteKey > b.siteKey ? 1 : 0))
620
+
621
+ // A site whose decline reason is `disjoint-dispatch` or `strategy-preempted` is
622
+ // NOT backlog: first-char dispatch and the two literal strategies already avoid
623
+ // the re-scan. Counting them would inflate the backlog with sites that are
624
+ // already in the good state.
625
+ const backlog = entries.filter(e =>
626
+ !e.factored && e.groups.length > 0
627
+ && e.declineReason !== 'disjoint-dispatch' && e.declineReason !== 'strategy-preempted')
628
+
629
+ return {
630
+ schema: 'parseman.choice-inventory/1',
631
+ rules: ruleMap.length,
632
+ choiceSites: entries.length,
633
+ factoredSites: entries.filter(e => e.factored).length,
634
+ backlogSites: backlog.length,
635
+ backlogArms: backlog.reduce((n, e) => n + e.unfactoredArms, 0),
636
+ unresolvedRoots,
637
+ entries,
638
+ }
639
+ }
640
+
641
+
642
+ // ── modelling the COMPILED first-char gate ───────────────────────────────────
643
+
644
+ /**
645
+ * THE BLIND SPOT THIS EXISTS TO CLOSE.
646
+ *
647
+ * The interpreter's `firstMatch` loop (src/combinators/choice.ts:149-165) enters every
648
+ * arm unconditionally. The EMITTED engine does NOT: `src/table/emit-assembly.ts`
649
+ * hoists a per-choice candidate mask and emits a per-arm first-CHARACTER guard, so an arm
650
+ * whose first set excludes the character at the current position is never entered at
651
+ * all. First-set gating is the project's single largest parse lever — 25-48% across all
652
+ * four jess parsers — and an instrument that measures the interpreter is blind to every
653
+ * byte of it.
654
+ *
655
+ * Unmodelled, this does not merely inflate a number, it REORDERS the ranking, because
656
+ * the inflation is proportional to how narrow an arm's first set is — exactly the arms
657
+ * codegen handles best. Measured on jess's `Value` choice: interpreted, arm 0
658
+ * `Percentage` is entered ~25,939 times and fails 97%, and reads as the top finding;
659
+ * compiled, it is gated on `+ - . 0-9` and 44% of those entries never happen. There is
660
+ * no gating gap there to close. Arm 4 `IdentBlock` — 3,743 of 3,743 attempts failed,
661
+ * 74% of that site's rescanned bytes — survives the correction and is the real finding.
662
+ *
663
+ * So both columns are reported and neither is derived from the other. The DELTA is
664
+ * itself a result: an arm expensive interpreted and cheap gated is ALREADY SOLVED by
665
+ * codegen, and a grammar author who reorders it is doing unpaid work.
666
+ *
667
+ * FIDELITY. This replicates codegen's own predicate rather than approximating it, and
668
+ * `test/unit/choice-cost.test.ts` compiles real grammars and asserts the model agrees
669
+ * with the emitted guards, arm by arm — because a hand-copy of another module's logic
670
+ * drifts silently, which is the same class of defect as the blind spot itself.
671
+ *
672
+ * Returns the guarding set, or `null` when codegen emits NO guard for this arm (the
673
+ * compiled parser always enters it, exactly as the interpreter does).
674
+ */
675
+ function compiledFirstCharGate(p: Combinator<unknown>): FirstSet | null {
676
+ const fs = p._meta.firstSet
677
+ // codegen: needsFirstSetGuard(p) === fs.kind !== 'any' && !canMatchEmptyAtStart(p)
678
+ if (fs.kind !== 'any' && !canMatchEmptyAtStart(p)) return fs
679
+ // codegen's deep-recovery branch: a cached `any` that came only from `ref()`s
680
+ // (which cache `any()` at construction) recovers a real guard from the deep,
681
+ // ref-resolving first set. This is also what the compose/linkable path resolves a
682
+ // DEFERRED `/*@FS:name@*/` placeholder to at fuse time.
683
+ if (fs.kind !== 'any' || canMatchEmptyAtStart(p)) return null
684
+ let deep: FirstSet
685
+ try { deep = firstSetOf(p) } catch { return null }
686
+ return deep.kind === 'ranges' ? deep : null
687
+ }
688
+
689
+ /**
690
+ * A verbatim replica of `canMatchEmptyAtStart`, which lived in the source lowering
691
+ * deleted in `37c57b5`. Nothing shares this definition today: it is the last copy.
692
+ *
693
+ * Deliberately SHALLOW, and that is not an approximation — it is the behaviour. A
694
+ * `node()`-wrapped nullable parser falls to `default: false`, so codegen treats it as
695
+ * non-nullable and DOES emit a guard for it. Using the public `matchesEmpty()` here
696
+ * instead looks more correct and models the wrong thing: it would report those arms as
697
+ * ungated when the shipped parser gates them. Verified against emitted output.
698
+ */
699
+ function canMatchEmptyAtStart(p: Combinator<unknown>): boolean {
700
+ const d = p._def as ParserDef
701
+ switch (d.tag) {
702
+ case 'regex': return regexMatchesEmpty(d.source, d.flags)
703
+ case 'optional': case 'many': return true
704
+ case 'transform': case 'label': case 'field':
705
+ return canMatchEmptyAtStart((d as unknown as { parser: Combinator<unknown> }).parser)
706
+ case 'literal': return d.value.length === 0
707
+ default: return false
708
+ }
709
+ }
710
+
711
+ /** Does the pattern match the empty string? Falls back to `true` (conservatively
712
+ * nullable, hence always-try) when the pattern cannot be compiled — mirroring codegen. */
713
+ function regexMatchesEmpty(source: string, flags: string): boolean {
714
+ try { return new RegExp(`^(?:${source})$`, flags.replace(/[gy]/g, '')).test('') }
715
+ catch { return true }
716
+ }
717
+
718
+ function inSet(code: number, fs: FirstSet): boolean {
719
+ if (fs.kind === 'any') return true
720
+ if (fs.kind === 'empty') return false
721
+ for (const r of fs.ranges) if (code >= r.lo && code <= r.hi) return true
722
+ return false
723
+ }
724
+
725
+ /** Exposed so the fidelity test can compare the model against emitted guards. */
726
+ export function modelledFirstCharGate(p: Combinator<unknown>): FirstSet | null {
727
+ return compiledFirstCharGate(p)
728
+ }
729
+
730
+ // ── HALF 2: corpus-driven wasted work ────────────────────────────────────────
731
+
732
+ /** Terminal tags. Reaching input position `q` means a terminal was ATTEMPTED at `q`,
733
+ * so bumping a high-water mark on terminal entry captures an attempt's furthest
734
+ * reach without inspecting any result. */
735
+ const TERMINAL_TAGS = new Set<ParserDef['tag']>(['literal', 'regex', 'keywords', 'scanTo'])
736
+
737
+ /** Strategies whose arms are tried in order through `def.parsers`, which is the same
738
+ * array object the choice's closure reads at call time — so substituting a slot
739
+ * installs instrumentation with no change to choice.ts and no flag on its hot path.
740
+ * `literalsLongestFirst` is excluded because it captures its sorted arm array at
741
+ * CONSTRUCTION time; a substituted slot would never be read, and instrumentation
742
+ * that silently does nothing is worse than none. */
743
+ const INSTRUMENTABLE = new Set<string>(['firstMatch', 'sharedPrefix'])
744
+
745
+ export type WastedWorkCorpusEntry = {
746
+ /** Stable identifier. Use a repo-relative path, never an absolute one — it appears
747
+ * in the report, and an absolute path would make the report machine-dependent. */
748
+ id: string
749
+ text: string
750
+ }
751
+
752
+ export type ProfileWastedWorkOptions = {
753
+ /** The combinator rule map, for site naming and instrumentation. */
754
+ rules: ReadonlyArray<readonly [string, Combinator<unknown>]>
755
+ /** The rule to start each parse at. A key of `rules`, or a combinator. */
756
+ entry: string | Combinator<unknown>
757
+ corpus: ReadonlyArray<WastedWorkCorpusEntry>
758
+ /** Passed through to `run()`. Must not contain anything machine-dependent. */
759
+ runOptions?: Record<string, unknown>
760
+ /**
761
+ * How to parse one corpus entry. Defaults to `run()` in interpreted mode.
762
+ *
763
+ * Overridable because a grammar can need a driver of its own — jess's Less dialect,
764
+ * for instance, requires `state: { source: input }` because its reducers read it —
765
+ * and because a caller who has already built a driver should not have to duplicate
766
+ * it here. It must be a plain interpreted parse; anything compiled defeats the
767
+ * instrumentation silently, since a compiled artifact never reads `def.parsers`.
768
+ */
769
+ runner?: (entry: Combinator<unknown>, input: string, options: Record<string, unknown>) => { ok: boolean }
770
+ /** Minimum attempts before an always-failing arm counts as an ordering inversion.
771
+ * Default 4 — below that, "failed every time" is a sample size, not a finding. */
772
+ inversionMinAttempts?: number
773
+ }
774
+
775
+ type ArmRecord = {
776
+ site: ChoiceSite
777
+ siteKey: string
778
+ arm: number
779
+ label: string
780
+ gate: FirstSet | null
781
+ attempts: number
782
+ failures: number
783
+ wastedBytes: number
784
+ gatedAttempts: number
785
+ gatedFailures: number
786
+ gatedWastedBytes: number
787
+ }
788
+
789
+ /**
790
+ * Parse `corpus` with `rules` in INTERPRETED mode, counting input bytes re-scanned
791
+ * by failed choice alternatives.
792
+ *
793
+ * Fails closed on every way of measuring nothing: an empty rule map, an unknown entry
794
+ * rule, an empty corpus, an empty corpus file, or a grammar in which no instrumentable
795
+ * choice site exists. Each of those would otherwise produce a report of zero wasted
796
+ * bytes, which reads as a clean grammar.
797
+ */
798
+ export function profileWastedWork(opts: ProfileWastedWorkOptions): WastedWorkReport {
799
+ const { rules, corpus } = opts
800
+ const { walked, unresolvedRoots } = walkRules(rules)
801
+
802
+ const entryParser = typeof opts.entry === 'string'
803
+ ? rules.find(([n]) => n === opts.entry)?.[1]
804
+ : opts.entry
805
+ if (entryParser === undefined) {
806
+ throw new TypeError(
807
+ `choice-cost: entry rule '${String(opts.entry)}' is not in the rule map `
808
+ + `(have: ${rules.map(([n]) => n).slice(0, 12).join(', ')}${rules.length > 12 ? ', …' : ''}).`,
809
+ )
810
+ }
811
+ if (corpus.length === 0) {
812
+ throw new TypeError('choice-cost: the corpus is EMPTY. A profile over no input measures nothing; refusing to report zero wasted bytes as a result.')
813
+ }
814
+ for (const f of corpus) {
815
+ if (f.text.length === 0) throw new TypeError(`choice-cost: corpus entry '${f.id}' is EMPTY.`)
816
+ }
817
+
818
+ // ── install ────────────────────────────────────────────────────────────────
819
+ // Two substitutions, both reverted in the `finally`:
820
+ // arms — `def.parsers[i]` is replaced by a delegating combinator that carries
821
+ // its own (site, index). Identity by SLOT, not by object, so an arm
822
+ // subtree shared between two sites is attributed to the right one.
823
+ // terminals — `parse` is replaced in place, because a terminal can sit in a
824
+ // single-child slot (`node.parser`) that the wrapper captured as a
825
+ // closure local and would never re-read from the def.
826
+ let highWater = 0
827
+ const armRecords: ArmRecord[] = []
828
+ const siteTotals = new Map<string, WastedWorkSite>()
829
+ const restore: (() => void)[] = []
830
+ // LIFO: terminal `parse` patches are pushed after the slot substitutions, and a
831
+ // terminal that also sits in a substituted slot must be unpatched before the slot is
832
+ // restored, or the original object would be handed back still wearing a patch.
833
+ const undoAll = (): void => { for (let i = restore.length - 1; i >= 0; i--) restore[i]!() }
834
+
835
+ let instrumentedSites = 0
836
+ let uninstrumentableSites = 0
837
+
838
+ // THE INSTALL IS COVERED TOO, not just the run. Both substitutions mutate a shared
839
+ // combinator tree, so anything that throws part-way through — `compiledFirstCharGate`,
840
+ // `armLabel`, the terminal patch loop — would otherwise leave orphaned instrumentation
841
+ // installed for the rest of the process and every later parse would run through it.
842
+ try {
843
+ for (const w of walked) {
844
+ if (w.d.tag !== 'choice') continue
845
+ const siteKey = choiceSiteKey(w.site)
846
+ const strategy = w.d.strategy.tag as ChoiceStrategyTag
847
+ const canInstrument = INSTRUMENTABLE.has(strategy) && !w.d.disjoint
848
+ const rec: WastedWorkSite = {
849
+ siteKey, site: w.site, strategy, arity: w.d.parsers.length,
850
+ instrumented: canInstrument, attempts: 0, failures: 0, wastedBytes: 0,
851
+ gatedAttempts: 0, gatedFailures: 0, gatedWastedBytes: 0,
852
+ }
853
+ // Two distinct choice INSTANCES can walk to the same site key only if the same
854
+ // slot were visited twice, which `seen` prevents. Keep the first and count the
855
+ // second as uninstrumentable rather than silently merging their numbers.
856
+ if (siteTotals.has(siteKey)) { uninstrumentableSites++; continue }
857
+ siteTotals.set(siteKey, rec)
858
+ if (!canInstrument) { uninstrumentableSites++; continue }
859
+ instrumentedSites++
860
+
861
+ const slots = w.d.parsers
862
+ for (let i = 0; i < slots.length; i++) {
863
+ const original = slots[i]!
864
+ // Computed ONCE per arm, outside the parse loop: the guard is a property of the
865
+ // grammar, not of a position, so recomputing it per attempt would be the profiler
866
+ // paying the cost it is measuring.
867
+ const gate = compiledFirstCharGate(original)
868
+ const armRec: ArmRecord = {
869
+ site: w.site, siteKey, arm: i, label: armLabel(original), gate,
870
+ attempts: 0, failures: 0, wastedBytes: 0,
871
+ gatedAttempts: 0, gatedFailures: 0, gatedWastedBytes: 0,
872
+ }
873
+ armRecords.push(armRec)
874
+ const wrapper: Combinator<unknown> = {
875
+ _tag: original._tag,
876
+ _meta: original._meta,
877
+ _def: original._def,
878
+ parse(input: string, pos: number, ctx: ParseContext): ParseResult<unknown> {
879
+ // Would COMPILED output have entered this arm at all? The arm is still run
880
+ // either way — skipping it would change what the grammar parses, and this is
881
+ // a measurement, not a mode. Only the accounting differs.
882
+ const code = pos < input.length ? (input.codePointAt(pos) ?? -1) : -1
883
+ const entered = gate === null || (code >= 0 && inSet(code, gate))
884
+
885
+ const saved = highWater
886
+ highWater = pos
887
+ const r = original.parse(input, pos, ctx)
888
+ const reach = highWater
889
+ highWater = saved > reach ? saved : reach
890
+
891
+ armRec.attempts++
892
+ rec.attempts++
893
+ if (entered) { armRec.gatedAttempts++; rec.gatedAttempts++ }
894
+ if (!r.ok) {
895
+ armRec.failures++
896
+ rec.failures++
897
+ if (entered) { armRec.gatedFailures++; rec.gatedFailures++ }
898
+ const wasted = reach - pos
899
+ if (wasted > 0) {
900
+ armRec.wastedBytes += wasted
901
+ rec.wastedBytes += wasted
902
+ if (entered) { armRec.gatedWastedBytes += wasted; rec.gatedWastedBytes += wasted }
903
+ }
904
+ }
905
+ return r
906
+ },
907
+ } as Combinator<unknown>
908
+ // `_ruleName` is read by site naming and by `deriveExpected`; carry it across so
909
+ // the substituted slot is indistinguishable from the original to everything but
910
+ // the counter.
911
+ const rn = ruleNameOf(original)
912
+ if (rn !== undefined) (wrapper as unknown as { _ruleName?: string })._ruleName = rn
913
+ slots[i] = wrapper
914
+ restore.push(() => { slots[i] = original })
915
+ }
916
+ }
917
+
918
+ if (instrumentedSites === 0) {
919
+ // No explicit undo here — the `catch` below owns it, in LIFO order.
920
+ throw new TypeError(
921
+ 'choice-cost: the grammar contains NO instrumentable choice site (no `firstMatch` or '
922
+ + '`sharedPrefix` choice). A wasted-work profile over it would report zero for a reason '
923
+ + 'that has nothing to do with the grammar being efficient, so it is refused.',
924
+ )
925
+ }
926
+
927
+ for (const w of walked) {
928
+ if (!TERMINAL_TAGS.has(w.d.tag)) continue
929
+ const p = w.p as { parse: Combinator<unknown>['parse'] }
930
+ const original = p.parse
931
+ p.parse = ((input: string, pos: number, ctx: ParseContext) => {
932
+ if (pos > highWater) highWater = pos
933
+ return original.call(w.p as never, input, pos, ctx)
934
+ }) as Combinator<unknown>['parse']
935
+ restore.push(() => { p.parse = original })
936
+ }
937
+ } catch (e) {
938
+ undoAll()
939
+ throw e
940
+ }
941
+
942
+ // ── run ────────────────────────────────────────────────────────────────────
943
+ let parsedOk = 0
944
+ let parsedFailed = 0
945
+ let corpusBytes = 0
946
+ try {
947
+ const runner = opts.runner ?? ((e, input, options) => run(e, input, options))
948
+ for (const f of corpus) {
949
+ corpusBytes += f.text.length
950
+ highWater = 0
951
+ const r = runner(entryParser, f.text, opts.runOptions ?? {})
952
+ if (r.ok) parsedOk++
953
+ else parsedFailed++
954
+ }
955
+ } finally {
956
+ undoAll()
957
+ }
958
+
959
+ const arms: WastedWorkArm[] = armRecords
960
+ .filter(a => a.attempts > 0)
961
+ .map(a => ({
962
+ siteKey: a.siteKey, site: a.site, arm: a.arm, label: a.label,
963
+ attempts: a.attempts, failures: a.failures, wastedBytes: a.wastedBytes,
964
+ firstCharGated: a.gate !== null,
965
+ gatedAttempts: a.gatedAttempts, gatedFailures: a.gatedFailures, gatedWastedBytes: a.gatedWastedBytes,
966
+ }))
967
+ // Ranked by the COMPILED column: what the shipped parser pays is the question a
968
+ // grammar author is actually asking. Interpreted bytes break the tie so the
969
+ // ordering stays total and the report stays diffable.
970
+ .sort((x, y) => y.gatedWastedBytes - x.gatedWastedBytes
971
+ || y.wastedBytes - x.wastedBytes
972
+ || (x.siteKey < y.siteKey ? -1 : x.siteKey > y.siteKey ? 1 : 0)
973
+ || x.arm - y.arm)
974
+
975
+ const sites = [...siteTotals.values()]
976
+ .sort((x, y) => y.gatedWastedBytes - x.gatedWastedBytes
977
+ || y.wastedBytes - x.wastedBytes
978
+ || (x.siteKey < y.siteKey ? -1 : x.siteKey > y.siteKey ? 1 : 0))
979
+
980
+ // "A later arm matched" is what separates an inversion from an arm that is simply
981
+ // never the right answer for this corpus. Without it, the last arm of every choice —
982
+ // which fails whenever the whole choice fails — would head the list.
983
+ const minAttempts = opts.inversionMinAttempts ?? 4
984
+ const inversions = arms
985
+ .filter(a =>
986
+ a.gatedAttempts >= minAttempts && a.gatedFailures === a.gatedAttempts
987
+ && arms.some(b => b.siteKey === a.siteKey && b.arm > a.arm && b.gatedAttempts > b.gatedFailures))
988
+ .sort((x, y) => y.gatedAttempts - x.gatedAttempts
989
+ || y.gatedWastedBytes - x.gatedWastedBytes
990
+ || (x.siteKey < y.siteKey ? -1 : x.siteKey > y.siteKey ? 1 : 0)
991
+ || x.arm - y.arm)
992
+
993
+ return {
994
+ schema: 'parseman.wasted-work/1',
995
+ corpusFiles: corpus.length,
996
+ corpusBytes,
997
+ parsedOk,
998
+ parsedFailed,
999
+ instrumentedSites,
1000
+ uninstrumentableSites,
1001
+ unresolvedRoots,
1002
+ totalWastedBytes: sites.reduce((n, s) => n + s.wastedBytes, 0),
1003
+ totalGatedWastedBytes: sites.reduce((n, s) => n + s.gatedWastedBytes, 0),
1004
+ arms,
1005
+ inversions,
1006
+ sites,
1007
+ }
1008
+ }