parseman 0.52.0 → 0.54.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/dist/analysis/diagnostics.js +636 -126
  2. package/dist/analysis/diagnostics.js.map +3 -3
  3. package/dist/cli/index.js +6191 -6274
  4. package/dist/cli/index.js.map +3 -3
  5. package/dist/combinators/field-trivia.d.ts +40 -0
  6. package/dist/combinators/field-trivia.d.ts.map +1 -0
  7. package/dist/combinators/map.d.ts +19 -2
  8. package/dist/combinators/map.d.ts.map +1 -1
  9. package/dist/combinators/parser.d.ts +34 -0
  10. package/dist/combinators/parser.d.ts.map +1 -1
  11. package/dist/combinators/ref.d.ts +0 -13
  12. package/dist/combinators/ref.d.ts.map +1 -1
  13. package/dist/combinators/repeat.d.ts.map +1 -1
  14. package/dist/combinators/sequence.d.ts.map +1 -1
  15. package/dist/compiler/borrowed-rules.d.ts +30 -0
  16. package/dist/compiler/borrowed-rules.d.ts.map +1 -0
  17. package/dist/compiler/compile-linkable-table.d.ts +1 -1
  18. package/dist/compiler/fields.d.ts +2 -0
  19. package/dist/compiler/fields.d.ts.map +1 -1
  20. package/dist/compiler/grammar-coverage-ids.d.ts.map +1 -1
  21. package/dist/compiler/ir-serialize.d.ts +10 -4
  22. package/dist/compiler/ir-serialize.d.ts.map +1 -1
  23. package/dist/compiler/linker.d.ts +48 -91
  24. package/dist/compiler/linker.d.ts.map +1 -1
  25. package/dist/coverage.d.ts +1 -1
  26. package/dist/coverage.d.ts.map +1 -1
  27. package/dist/cst/reflection.d.ts.map +1 -1
  28. package/dist/cst/trivia-charscan.d.ts +11 -3
  29. package/dist/cst/trivia-charscan.d.ts.map +1 -1
  30. package/dist/cst/trivia-kinds.d.ts +11 -0
  31. package/dist/cst/trivia-kinds.d.ts.map +1 -1
  32. package/dist/index.d.ts +1 -0
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js +764 -888
  35. package/dist/index.js.map +3 -3
  36. package/dist/language-service/index.js.map +1 -1
  37. package/dist/plugin/evaluator.d.ts +38 -2
  38. package/dist/plugin/evaluator.d.ts.map +1 -1
  39. package/dist/plugin/index.d.ts.map +1 -1
  40. package/dist/plugin/index.js +1443 -729
  41. package/dist/plugin/index.js.map +3 -3
  42. package/dist/plugin/ir-eval.d.ts +21 -0
  43. package/dist/plugin/ir-eval.d.ts.map +1 -0
  44. package/dist/plugin/reducer-resolver.d.ts +6 -0
  45. package/dist/plugin/reducer-resolver.d.ts.map +1 -1
  46. package/dist/spec/index.js +555 -6109
  47. package/dist/spec/index.js.map +3 -3
  48. package/dist/spec/railroad-lib.d.ts +7 -2
  49. package/dist/spec/railroad-lib.d.ts.map +1 -1
  50. package/dist/spec/railroad.d.ts.map +1 -1
  51. package/dist/table/assemble.d.ts +9 -0
  52. package/dist/table/assemble.d.ts.map +1 -1
  53. package/dist/table/compile-rule-map.d.ts +11 -4
  54. package/dist/table/compile-rule-map.d.ts.map +1 -1
  55. package/dist/table/compile.d.ts +16 -0
  56. package/dist/table/compile.d.ts.map +1 -1
  57. package/dist/table/emit-assembly.d.ts +1 -1
  58. package/dist/table/emit-assembly.d.ts.map +1 -1
  59. package/dist/table/emit.d.ts +6 -2
  60. package/dist/table/emit.d.ts.map +1 -1
  61. package/dist/table/encode.d.ts.map +1 -1
  62. package/dist/table/exec.d.ts.map +1 -1
  63. package/dist/table/field-spec.d.ts +13 -0
  64. package/dist/table/field-spec.d.ts.map +1 -0
  65. package/dist/table/index.js +480 -199
  66. package/dist/table/index.js.map +2 -2
  67. package/dist/table/scalar-terminal.d.ts +11 -0
  68. package/dist/table/scalar-terminal.d.ts.map +1 -1
  69. package/dist/types.d.ts +14 -0
  70. package/dist/types.d.ts.map +1 -1
  71. package/dist/version.d.ts +1 -1
  72. package/package.json +1 -1
  73. package/src/analysis/duplication.ts +1 -1
  74. package/src/analysis/rebuild.ts +1 -1
  75. package/src/combinators/field-trivia.ts +44 -0
  76. package/src/combinators/map.ts +48 -6
  77. package/src/combinators/parser.ts +133 -67
  78. package/src/combinators/ref.ts +10 -1
  79. package/src/combinators/repeat.ts +6 -1
  80. package/src/combinators/sequence.ts +9 -1
  81. package/src/compiler/borrowed-rules.ts +88 -0
  82. package/src/compiler/compile-linkable-table.ts +1 -1
  83. package/src/compiler/fields.ts +5 -2
  84. package/src/compiler/grammar-coverage-ids.ts +35 -6
  85. package/src/compiler/ir-serialize.ts +19 -119
  86. package/src/compiler/linker.ts +190 -666
  87. package/src/coverage.ts +7 -5
  88. package/src/cst/reflection.ts +6 -1
  89. package/src/cst/trivia-charscan.ts +17 -56
  90. package/src/cst/trivia-kinds.ts +24 -8
  91. package/src/index.ts +1 -0
  92. package/src/plugin/evaluator.ts +114 -14
  93. package/src/plugin/index.ts +588 -110
  94. package/src/plugin/ir-eval.ts +165 -0
  95. package/src/plugin/reducer-resolver.ts +20 -3
  96. package/src/spec/railroad-lib.ts +18 -8
  97. package/src/spec/railroad.ts +3 -4
  98. package/src/table/assemble.ts +126 -22
  99. package/src/table/compile-rule-map.ts +18 -8
  100. package/src/table/compile.ts +61 -0
  101. package/src/table/emit-assembly.ts +38 -18
  102. package/src/table/emit.ts +15 -4
  103. package/src/table/encode.ts +18 -2
  104. package/src/table/exec.ts +15 -1
  105. package/src/table/field-spec.ts +17 -0
  106. package/src/table/scalar-terminal.ts +25 -0
  107. package/src/types.ts +12 -2
  108. package/src/version.ts +1 -1
@@ -0,0 +1,165 @@
1
+ /**
2
+ * The macro plugin's IR evaluator: turns a composed grammar's carried IR back
3
+ * into combinators so the build can re-lower it. Build time only — see below.
4
+ */
5
+ import type { Combinator } from '../types.ts'
6
+ import { rules } from '../combinators/parser.ts'
7
+ import { ref } from '../combinators/ref.ts'
8
+ import { regex } from '../combinators/regex.ts'
9
+ import { literal } from '../combinators/literal.ts'
10
+ import { keywords } from '../combinators/keywords.ts'
11
+ import { sequence } from '../combinators/sequence.ts'
12
+ import { choice } from '../combinators/choice.ts'
13
+ import { dispatch, endsWith, matches, otherwise, routed, startsWith, when } from '../combinators/dispatch.ts'
14
+ import { attempt } from '../combinators/attempt.ts'
15
+ import { many, oneOrMore, optional, sepBy, keepSeparator } from '../combinators/repeat.ts'
16
+ import { not } from '../combinators/not.ts'
17
+ import { peek } from '../combinators/peek.ts'
18
+ import { node } from '../combinators/node.ts'
19
+ import { parser } from '../combinators/grammar.ts'
20
+ import { scanTo, balanced } from '../combinators/scanTo.ts'
21
+ import { token, leaf, sourceLeaf } from '../combinators/token.ts'
22
+ import { classifiedTrivia, transform, trivia, label, field } from '../combinators/map.ts'
23
+ import { expect as expectC } from '../combinators/expect.ts'
24
+ import { withCtx } from '../combinators/withCtx.ts'
25
+ import { adjacent, notAdjacent } from '../combinators/adjacency.ts'
26
+
27
+ type Comb = Combinator<unknown>
28
+
29
+ /** Reconstruct a rule map from serialized IR (the inverse of `serializeRuleMap`) —
30
+ * evaluate the combinator-construction expression with every constructor in scope.
31
+ *
32
+ * BUILD TIME ONLY. It evaluates source text, so it lives in the plugin, which runs
33
+ * in the bundler; no runtime module imports it
34
+ * (docs/design/runtime-and-size-contract.md, rule 1). */
35
+ export function evalRuleMapIR(ir: string): Array<[string, Comb]> {
36
+ return Object.entries(evalRulesIR(ir))
37
+ }
38
+
39
+ /** The `rules()` grammar an IR expression builds — the map plus, non-enumerable,
40
+ * the recipe (`RULES_RECIPE`) a composition links it by. Build time only, as above. */
41
+ export function evalRulesIR(ir: string): Record<string, Comb> {
42
+ return evalIR(ir, rules) as Record<string, Comb>
43
+ }
44
+
45
+ /** The inverse of `serializeCombinator`: the same evaluation, with `rules` reading the
46
+ * one `scanSkip` unit back instead of building a grammar around it. Build time only. */
47
+ export function evalCombinatorIR(ir: string): Comb | null {
48
+ const units = evalIR(ir, (opts: { scanSkip?: Comb[] }) => opts.scanSkip) as Comb[] | undefined
49
+ return units?.length === 1 ? units[0]! : null
50
+ }
51
+
52
+ function evalIR(ir: string, rulesImpl: unknown): unknown {
53
+ // `_tf`/`_nd` reconstruct a transform/node AND restore its captured callback
54
+ // source (`_def.fnSrc`/`buildSrc`) so re-lowering inlines it statically. The live
55
+ // fn is only needed for interpreted mode; a self-contained transform source is
56
+ // eval'd, a node build (which may reference imported AST classes) is left to its
57
+ // source only. Macro-only validation metadata is carried as plain data: this
58
+ // runtime module never parses callback source or imports a compiler frontend.
59
+ const _tf = (child: Comb, src: string, recognitionOnly = false): Comb => {
60
+ let fn: (...a: unknown[]) => unknown
61
+ // eslint-disable-next-line no-eval
62
+ try { fn = (0, eval)(`(${src})`) } catch { fn = () => { throw new Error('IR transform fn not materialized') } }
63
+ const t = transform(child as never, fn as never)
64
+ ;(t._def as { fnSrc?: string }).fnSrc = src
65
+ if (recognitionOnly) (t._def as { recognitionOnly?: boolean }).recognitionOnly = true
66
+ return t as Comb
67
+ }
68
+ const _lf = (child: Comb, src: string, source = false): Comb => {
69
+ let fn: (...a: unknown[]) => unknown
70
+ // eslint-disable-next-line no-eval
71
+ try { fn = (0, eval)(`(${src})`) } catch { fn = () => { throw new Error('IR leaf fn not materialized') } }
72
+ const l = source ? sourceLeaf(child as never, fn as never) : leaf(child as never, fn as never)
73
+ ;(l._def as { fnSrc?: string }).fnSrc = src
74
+ return l as Comb
75
+ }
76
+ const _nd = (type: string, child: Comb, src: string, opts?: unknown, staticError?: readonly string[], sigSrc?: string, buildImports?: ReadonlyArray<{ local: string; source: string; imported: string }>, rawUnused?: true): Comb => {
77
+ if (staticError !== undefined && staticError.length > 0) {
78
+ // Fail closed: a builder with an un-rescuable binding is NOT fused. The
79
+ // plugin catches this and leaves the runtime compose() in place rather than
80
+ // shipping a fused table that would ReferenceError on import. Import-rescued
81
+ // free names never reach here — they ride in `buildImports`, not `staticError`.
82
+ throw new Error(`IR direct node builder for ${type} must be macro-static and self-contained; unsupported binding(s): ${staticError.join(', ')}`)
83
+ }
84
+ // A serialized direct builder needs an inert sentinel as well as buildSrc.
85
+ // `node(..., undefined)` is structural, so re-lowering a composed artifact
86
+ // silently routes it through ctx.build/default CST even though the IR still
87
+ // carries the callback source. The compiler is the only consumer that may
88
+ // materialize `buildSrc`; raw IR interpretation deliberately rejects direct
89
+ // builders rather than evaluating arbitrary captured source at runtime.
90
+ const n = node(type, child as never, (() => { throw new Error('IR node build requires static re-lowering') }) as never, opts as never)
91
+ ;(n._def as { buildSrc?: string; buildStaticError?: readonly string[] }).buildSrc = src
92
+ if (staticError !== undefined) (n._def as { buildStaticError?: readonly string[] }).buildStaticError = staticError
93
+ // Analysis-only resolved reducer signature (see `buildAnalysisSrc`). Without it a
94
+ // re-lowered composed artifact silently re-acquires the fail-open capture cost the
95
+ // authoring module had already resolved away.
96
+ if (sigSrc !== undefined) (n._def as { buildSigSrc?: string }).buildSigSrc = sigSrc
97
+ if (rawUnused === true) (n._def as { buildRawUnused?: true }).buildRawUnused = true
98
+ // Re-attach the direct-builder import provenance so the plugin's re-lower pass
99
+ // can re-emit the imports into the consuming module. Plain data — this runtime
100
+ // module never resolves or emits imports itself.
101
+ if (buildImports !== undefined) (n._def as { buildImports?: ReadonlyArray<{ local: string; source: string; imported: string }> }).buildImports = buildImports
102
+ return n as Comb
103
+ }
104
+ // `_gch` rebuilds a GATED choice AND restores its `_def.gateSrcs` (parallel to the
105
+ // arms) — the gate mirror of `_tf`/`_nd`. Each item is either a plain arm or a
106
+ // `[gateSource, arm]` tuple: the source is eval'd to the live predicate (interpreted
107
+ // mode) and recorded as the arm's gateSrcs entry so re-lowering INLINES the gate
108
+ // statically (keeping the artifact fusible via `emitFusedSource`). A missing/failed
109
+ // source falls back to a throwing predicate — the same contract as `_tf`.
110
+ const _gch = (items: Array<Comb | [string, Comb]>): Comb => {
111
+ const gateSrcs: (string | null)[] = []
112
+ const arms = items.map(it => {
113
+ if (Array.isArray(it)) {
114
+ const [src, comb] = it
115
+ gateSrcs.push(src)
116
+ let gate: (s: unknown) => boolean
117
+ // eslint-disable-next-line no-eval
118
+ try { gate = (0, eval)(`(${src})`) } catch { gate = () => { throw new Error('IR gate fn not materialized') } }
119
+ return { gate, combinator: comb }
120
+ }
121
+ gateSrcs.push(null)
122
+ return it
123
+ })
124
+ const c = (choice as (...a: unknown[]) => Comb)(...arms)
125
+ // `choice` is the authority on gate alignment; `gateSrcs` must line up 1:1 with
126
+ // the constructed `_def.gates`. They always do on the normal path (both from the
127
+ // same `items.map`), but assert it so a future change in `choice` that coalesces
128
+ // or drops arms can't silently ship a mis-aligned gate-source array.
129
+ const gates = (c._def as { gates?: unknown[] }).gates
130
+ if (gates && gates.length !== gateSrcs.length) {
131
+ throw new Error(`_gch: gateSrcs length ${gateSrcs.length} != choice gates length ${gates.length}`)
132
+ }
133
+ ;(c._def as { gateSrcs?: (string | null)[] }).gateSrcs = gateSrcs
134
+ return c
135
+ }
136
+ // `_wc` rebuilds a `withCtx` AND restores its `_def.extraSrc` (the source of the
137
+ // `extra`/state value) — the withCtx mirror of `_tf`/`_nd`/`_gch`. The source is
138
+ // eval'd to the live `extra` value (interpreted mode) and recorded on the def so
139
+ // re-lowering INLINES the state getter statically (`() => (extraSrc)`), keeping the
140
+ // artifact fusible via `emitFusedSource`. A plain `withCtx(value, inner)` would
141
+ // leave `extraSrc` unset → codegen emits a source-less runtime closure (a non-static
142
+ // callback) → `emitFusedSource` fails and a downstream `compose()` silently falls
143
+ // back to a runtime fuse. A missing/failed source falls back to `undefined` state —
144
+ // the same best-effort contract as `_tf`.
145
+ const _wc = (src: string, inner: Comb): Comb => {
146
+ let extra: unknown
147
+ // eslint-disable-next-line no-eval
148
+ try { extra = (0, eval)(`(${src})`) } catch { extra = undefined }
149
+ const w = (withCtx as (e: unknown, c: Comb) => Comb)(extra, inner)
150
+ ;(w._def as { extraSrc?: string }).extraSrc = src
151
+ return w
152
+ }
153
+ // eslint-disable-next-line no-new-func
154
+ const fn = new Function(
155
+ 'rules', 'ref', 'regex', 'literal', 'keywords', 'sequence', 'choice', 'dispatch', 'when', 'startsWith', 'endsWith', 'matches', 'otherwise', 'routed', 'attempt',
156
+ 'many', 'oneOrMore', 'optional', 'sepBy', 'keepSeparator', 'not', 'peek', 'node', 'parser',
157
+ 'scanTo', 'balanced', 'token', 'leaf', 'transform', 'trivia', 'classifiedTrivia', 'label', 'field', 'expect', 'adjacent', 'notAdjacent', '_tf', '_lf', '_nd', '_gch', '_wc',
158
+ `return (${ir})`,
159
+ )
160
+ return fn(
161
+ rulesImpl, ref, regex, literal, keywords, sequence, choice, dispatch, when, startsWith, endsWith, matches, otherwise, routed, attempt,
162
+ many, oneOrMore, optional, sepBy, keepSeparator, not, peek, node, parser,
163
+ scanTo, balanced, token, leaf, transform, trivia, classifiedTrivia, label, field, expectC, adjacent, notAdjacent, _tf, _lf, _nd, _gch, _wc,
164
+ )
165
+ }
@@ -118,8 +118,11 @@ function patternNames(pat: unknown, out: string[]): void {
118
118
  }
119
119
 
120
120
  function declare(scope: ScopeNode, name: string, b: Binding): void {
121
- // First declaration wins; a duplicate `var` re-declaration does not change the shape.
122
- if (!scope.bindings.has(name)) scope.bindings.set(name, b)
121
+ // First declaration wins the shape, but a second one (`var x = a; … var x = b`, a
122
+ // repeated function) can rebind the name, so it counts as a reassignment.
123
+ const prev = scope.bindings.get(name)
124
+ if (prev) prev.reassigned = true
125
+ else scope.bindings.set(name, b)
123
126
  }
124
127
 
125
128
  /** `var` and function declarations hoist to the nearest FUNCTION scope, not the block. */
@@ -238,7 +241,10 @@ function markReassignments(body: unknown[], root: ScopeNode): void {
238
241
  if (Array.isArray(n)) { for (const item of n) walk(item); return }
239
242
  const node = n as AnyNode
240
243
  if (typeof node.type !== 'string') return
241
- if (node.type === 'AssignmentExpression' || node.type === 'UpdateExpression') {
244
+ // `for (x of …)` / `for (x in …)` writes `x` too when its head declares nothing.
245
+ const loopWrite = (node.type === 'ForOfStatement' || node.type === 'ForInStatement')
246
+ && (node.left as AnyNode | undefined)?.type !== 'VariableDeclaration'
247
+ if (node.type === 'AssignmentExpression' || node.type === 'UpdateExpression' || loopWrite) {
242
248
  const target = (node.type === 'UpdateExpression' ? node.argument : node.left) as AnyNode | undefined
243
249
  const names: string[] = []
244
250
  if (target) patternNames(target, names)
@@ -255,6 +261,17 @@ function markReassignments(body: unknown[], root: ScopeNode): void {
255
261
  for (const stmt of body) walk(stmt)
256
262
  }
257
263
 
264
+ /**
265
+ * Whether a top-level binding of a module is never reassigned or redeclared, so a `let` or
266
+ * `var` holds its initializer exactly as a `const` would. Bundlers emit top-level `const`
267
+ * as `var` (esbuild does), which is how a published entry's exports usually arrive.
268
+ */
269
+ export function topLevelIsStable(body: unknown[], moduleEnd: number): (name: string) => boolean {
270
+ const root = buildScopeTree(body, moduleEnd)
271
+ markReassignments(body, root)
272
+ return name => root.bindings.get(name)?.reassigned !== true
273
+ }
274
+
258
275
  // ---------------------------------------------------------------------------
259
276
  // Parameter list → arity, from the AST
260
277
  // ---------------------------------------------------------------------------
@@ -2,12 +2,18 @@
2
2
  * Vendored copy of tabatkins railroad-diagrams (v1.0.0), license CC0-1.0.
3
3
  * https://github.com/tabatkins/railroad-diagrams
4
4
  *
5
- * Public domain (CC0) — embedded verbatim as strings so the generated
6
- * railroad-diagram HTML page is fully self-contained (no CDN, no runtime dep).
5
+ * Public domain (CC0). The library is held ONCE, as real code: `railroadLibrary`
6
+ * renders SVG in Node, and `RAILROAD_JS` embeds the same function's own source in
7
+ * the generated HTML page (no CDN, no runtime dep). Nothing evaluates a string —
8
+ * docs/design/runtime-and-size-contract.md, rule 1.
7
9
  * Do not edit by hand; regenerate from the npm package if bumping versions.
8
10
  */
9
11
  /* eslint-disable */
10
- export const RAILROAD_JS: string = `/*
12
+ // @ts-nocheck — vendored third-party JavaScript, kept verbatim.
13
+
14
+ /** Install the railroad-diagrams builders onto `exports`. */
15
+ export function railroadLibrary(exports) {
16
+ /*
11
17
  Railroad Diagrams
12
18
  by Tab Atkins Jr. (and others)
13
19
  http://xanthir.com
@@ -90,7 +96,7 @@ At runtime, these constants can be found on the Diagram class.
90
96
  };
91
97
  FakeSVG.prototype.escapeString = function(string) {
92
98
  // Escape markdown and HTML special characters
93
- return string.replace(/[*_\\\`\\[\\]<&]/g, function(charString) {
99
+ return string.replace(/[*_\`\[\]<&]/g, function(charString) {
94
100
  return '&#' + charString.charCodeAt(0) + ';';
95
101
  });
96
102
  };
@@ -112,7 +118,7 @@ At runtime, these constants can be found on the Diagram class.
112
118
  str += ' ' + attr + '="' + (this.attrs[attr]+'').replace(/&/g, '&amp;').replace(/"/g, '&quot;') + '"';
113
119
  }
114
120
  str += '>';
115
- if(group) str += "\\n";
121
+ if(group) str += "\n";
116
122
  if(typeof this.children == 'string') {
117
123
  str += FakeSVG.prototype.escapeString(this.children);
118
124
  } else {
@@ -120,7 +126,7 @@ At runtime, these constants can be found on the Diagram class.
120
126
  str += e;
121
127
  });
122
128
  }
123
- str += '</' + this.tagName + '>\\n';
129
+ str += '</' + this.tagName + '>\n';
124
130
  return str;
125
131
  }
126
132
 
@@ -522,7 +528,7 @@ At runtime, these constants can be found on the Diagram class.
522
528
  */
523
529
  ['Diagram', 'ComplexDiagram', 'Sequence', 'Choice', 'Optional', 'OneOrMore', 'ZeroOrMore', 'Terminal', 'NonTerminal', 'Comment', 'Skip']
524
530
  .forEach(function(e,i) { root[e] = temp[i]; });
525
- }).call(this,
531
+ }).call(undefined,
526
532
  {
527
533
  VERTICAL_SEPARATION: 8,
528
534
  ARC_RADIUS: 10,
@@ -531,7 +537,11 @@ At runtime, these constants can be found on the Diagram class.
531
537
  INTERNAL_ALIGNMENT: 'center',
532
538
  }
533
539
  );
534
- `
540
+
541
+ }
542
+
543
+ /** The same library as page script: installs its builders as browser globals. */
544
+ export const RAILROAD_JS: string = `(${railroadLibrary.toString()})(window);`
535
545
 
536
546
  export const RAILROAD_CSS: string = `svg.railroad-diagram {
537
547
  background-color: hsl(30,20%,95%);
@@ -8,7 +8,7 @@
8
8
  */
9
9
  import type { Production, SpecModel, SpecNode } from './model.ts'
10
10
  import { renderExpr } from './ebnf.ts'
11
- import { RAILROAD_CSS, RAILROAD_JS } from './railroad-lib.ts'
11
+ import { RAILROAD_CSS, RAILROAD_JS, railroadLibrary } from './railroad-lib.ts'
12
12
 
13
13
  const j = (s: string): string => JSON.stringify(s)
14
14
 
@@ -69,7 +69,7 @@ function toDsl(node: SpecNode): string {
69
69
  }
70
70
  }
71
71
 
72
- // Vendored railroad-diagrams builders, evaluated once. `.toString()` on the
72
+ // Vendored railroad-diagrams builders, installed once. `.toString()` on the
73
73
  // result walks an in-memory FakeSVG tree and emits SVG markup with no DOM — so a
74
74
  // diagram can be rendered to a static SVG string at build time (Node), not just
75
75
  // in the browser. The library exports onto `exports` when it's an object.
@@ -89,8 +89,7 @@ let _builders: RailroadBuilders | null = null
89
89
  function builders(): RailroadBuilders {
90
90
  if (_builders) return _builders
91
91
  const mod = {} as Record<string, unknown>
92
- // eslint-disable-next-line no-new-func
93
- new Function('exports', RAILROAD_JS)(mod)
92
+ railroadLibrary(mod)
94
93
  _builders = mod as unknown as RailroadBuilders
95
94
  return _builders
96
95
  }
@@ -66,6 +66,7 @@
66
66
  import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserDef } from '../types.ts'
67
67
  import { balanced, scanTo } from '../combinators/scanTo.ts'
68
68
  import { buildFieldMap } from '../compiler/fields.ts'
69
+ import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA, LEAD, triviaFieldCapture } from '../combinators/field-trivia.ts'
69
70
  import { asciiFoldKey } from '../combinators/dispatch.ts'
70
71
  import { projectChild, unwrapChild } from '../combinators/node.ts'
71
72
  import { cstOutputHost } from '../compiler/build-arity.ts'
@@ -129,7 +130,7 @@ import { computeSiteLabels, reachableSites } from './site-labels.ts'
129
130
  import { refuseUnclassifiedRootScope } from '../cst/root-trivia-scope.ts'
130
131
  import { captureError, firstSetSentinel, matchesAt, orSentinel, recoverScan } from '../recovery/scan.ts'
131
132
  import {
132
- leadingLiteralFamily, leadingScalarSequence, leadingScalarTerminal, makeScalarRecognizer, scalarTerminalNodeChild,
133
+ largeChoiceLiteralPretest, leadingScalarSequence, leadingScalarTerminal, makeScalarRecognizer, scalarTerminalNodeChild,
133
134
  scalarTerminalNotChild, type ScalarRecognizer,
134
135
  } from './scalar-terminal.ts'
135
136
 
@@ -146,6 +147,7 @@ const EMIT_ENABLED = (globalThis as { process?: { env?: Record<string, string |
146
147
 
147
148
  /** Failure sentinel — SHARED with the other two engines, see `cell.ts`. */
148
149
  import { FAIL, newEndCell } from './cell.ts'
150
+ import { fieldSpec, leadingTriviaFieldAt } from './field-spec.ts'
149
151
 
150
152
  const EMPTY_TL: readonly number[] = Object.freeze([])
151
153
  /** `finishCstBuf`'s two empty sentinels, held here so the node piece can inline it. */
@@ -1053,6 +1055,15 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
1053
1055
  return cur
1054
1056
  }
1055
1057
 
1058
+ /**
1059
+ * A `field(…, { leadingTrivia: true })` term. The pre-scan cursor reaches the
1060
+ * field AFTER `step`'s scan, right before the field runs, so a field inside the
1061
+ * trivia grammar cannot take it first. One closure per call; only such terms pay.
1062
+ */
1063
+ function leadingFieldTerm(step: typeof nextTerm, child: Piece, input: string, cur: number, ctx: ParseContext): number {
1064
+ return step((i, p, c) => { LEAD.at = cur; return child(i, p, c) }, input, cur, ctx)
1065
+ }
1066
+
1056
1067
  function committed(c: ParseContext): boolean {
1057
1068
  return c._fc === true
1058
1069
  }
@@ -1793,12 +1804,27 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
1793
1804
  }
1794
1805
 
1795
1806
  case OP_FIELD: {
1796
- const name = k[code[ip + 1]!] as string
1807
+ const { name, bits } = fieldSpec(k[code[ip + 1]!])
1797
1808
  const child = link(code[ip + 2]!)
1809
+ if (bits === 0) {
1810
+ return (input, pos, ctx) => {
1811
+ const v = child(input, pos, ctx)
1812
+ if (v === FAIL) return FAIL
1813
+ ctx._fields?.push({ name, value: v, span: { start: pos, end: EC.e } })
1814
+ return v
1815
+ }
1816
+ }
1817
+ const leading = (bits & FIELD_LEADING_TRIVIA) !== 0
1818
+ const trailing = (bits & FIELD_TRAILING_TRIVIA) !== 0
1798
1819
  return (input, pos, ctx) => {
1820
+ const leadFrom = leading ? LEAD.at : -1
1821
+ LEAD.at = -1
1799
1822
  const v = child(input, pos, ctx)
1800
1823
  if (v === FAIL) return FAIL
1801
- ctx._fields?.push({ name, value: v, span: { start: pos, end: EC.e } })
1824
+ const end = EC.e
1825
+ const trailEnd = trailing && ctx.trivia !== undefined ? consumeTrivia(input, end, ctx) : end
1826
+ ctx._fields?.push(triviaFieldCapture(name, v, input, pos, end, leadFrom, trailEnd))
1827
+ EC.e = trailEnd
1802
1828
  return v
1803
1829
  }
1804
1830
  }
@@ -2322,6 +2348,12 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
2322
2348
  const kidIp = code[base + i]!
2323
2349
  if (code[kidIp] !== OP_ADJ) {
2324
2350
  const kid = link(kidIp)
2351
+ if (leadingTriviaFieldAt(code, k, kidIp)) {
2352
+ return (input, cur, ctx, inherited) => {
2353
+ ctx._sync = sync ?? inherited
2354
+ return leadingFieldTerm(step, kid, input, cur, ctx)
2355
+ }
2356
+ }
2325
2357
  return (input, cur, ctx, inherited) => {
2326
2358
  ctx._sync = sync ?? inherited
2327
2359
  return step(kid, input, cur, ctx)
@@ -2389,14 +2421,22 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
2389
2421
  }
2390
2422
  }
2391
2423
 
2424
+ // A `field(…, { leadingTrivia: true })` term is a boundary fact too: the
2425
+ // runner hands it the cursor from BEFORE the scan (`LEAD`, cell.ts).
2392
2426
  let hasAdj = false
2393
- for (let i = 1; i < n; i++) if (code[code[base + i]!] === OP_ADJ) { hasAdj = true; break }
2427
+ for (let i = 1; i < n; i++) {
2428
+ const kidIp = code[base + i]!
2429
+ if (code[kidIp] === OP_ADJ || leadingTriviaFieldAt(code, k, kidIp)) { hasAdj = true; break }
2430
+ }
2394
2431
  if (hasAdj) {
2395
2432
  const first = link(code[base]!)
2396
2433
  const runnerAt = (i: number): TermRunner => {
2397
2434
  const kidIp = code[base + i]!
2398
2435
  if (code[kidIp] !== OP_ADJ) {
2399
2436
  const kid = link(kidIp)
2437
+ if (leadingTriviaFieldAt(code, k, kidIp)) {
2438
+ return (input, cur, ctx) => leadingFieldTerm(step, kid, input, cur, ctx)
2439
+ }
2400
2440
  return (input, cur, ctx) => step(kid, input, cur, ctx)
2401
2441
  }
2402
2442
  const negated = code[kidIp + 1] === 1
@@ -2791,7 +2831,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
2791
2831
  if (n === 2 || n === 3) {
2792
2832
  const a0 = link(code[base]!), a1 = link(code[base + 1]!)
2793
2833
  const a2 = n === 3 ? link(code[base + 2]!) : undefined
2794
- const predecide = !hostCst && !REC && !cfg.probe && !cfg.coverage && !cfg.trackLines
2834
+ const predecide = !REC && !cfg.probe && !cfg.coverage && !cfg.trackLines
2795
2835
  const p0ip = predecide ? leadingScalarTerminal(code, code[base]!, 2, true, true) : -1
2796
2836
  const p1ip = predecide ? leadingScalarTerminal(code, code[base + 1]!, 2, true, true) : -1
2797
2837
  const p2ip = predecide && n === 3 ? leadingScalarTerminal(code, code[base + 2]!, 2, true, true) : -1
@@ -2958,6 +2998,56 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
2958
2998
  )
2959
2999
  }
2960
3000
  const masked = bindMasked(0, EMPTY_EXPECTED_PREFIX)
3001
+ // LITERAL PRETESTS. First-char dispatch cannot tell `@{…}` or `@@x` from
3002
+ // `@x`, so every arm sharing the char is entered, and under a CST host
3003
+ // each entered node opens a capture frame before failing on the second
3004
+ // char. An arm that must begin with a literal of two or more chars (or
3005
+ // one of a family of them) is tested on the literal first and, on a
3006
+ // miss, skipped exactly as a char-excluded arm is: its static expected
3007
+ // set is the one reported. The test is pure, so a miss leaves no
3008
+ // capture, trivia or field to roll back.
3009
+ const pBit: number[] = []
3010
+ const pRec: ScalarRecognizer[] = []
3011
+ if (!REC && !cfg.probe && !cfg.coverage && !cfg.trackLines) {
3012
+ for (let i = 0; i < n; i++) {
3013
+ for (const terminal of largeChoiceLiteralPretest(code, k, code[base + i]!) ?? []) {
3014
+ pBit.push(1 << i)
3015
+ pRec.push(scalarFor(terminal)!)
3016
+ }
3017
+ }
3018
+ }
3019
+ const pn = pBit.length
3020
+ if (pn !== 0) {
3021
+ return (input, pos, ctx) => {
3022
+ const c = lead(input, pos)
3023
+ const need = markCst(ctx)
3024
+ const mRaw = MRAW
3025
+ const mTl = MTL
3026
+ const mLv = MLV
3027
+ const mFl = need ? ctx._fields?.length ?? 0 : 0
3028
+ const mEr = need ? ctx._errors?.length ?? 0 : 0
3029
+ const mLog = need ? ctx._triviaLog?.length ?? 0 : 0
3030
+ const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
3031
+ if (c < 128) {
3032
+ const bits = mask[c < 0 ? 128 : c]!
3033
+ let tested = 0
3034
+ let hit = 0
3035
+ for (let i = 0; i < pn; i++) {
3036
+ const b = pBit[i]!
3037
+ if ((bits & b) === 0 || (hit & b) !== 0) continue
3038
+ tested |= b
3039
+ if (pRec[i]!(input, pos) >= 0) hit |= b
3040
+ }
3041
+ return masked(
3042
+ input, pos, ctx, bits & (~tested | hit),
3043
+ need, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot, undefined, 0, pos,
3044
+ )
3045
+ }
3046
+ return general(
3047
+ input, pos, ctx, c, need, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot, undefined, 0, pos,
3048
+ )
3049
+ }
3050
+ }
2961
3051
  return (input, pos, ctx) => {
2962
3052
  const c = lead(input, pos)
2963
3053
  const need = markCst(ctx)
@@ -3100,6 +3190,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
3100
3190
  const max = code[ip + 3]!
3101
3191
  const sepIp = code[ip + 4]!
3102
3192
  const sep = sepIp >= 0 ? link(sepIp) : undefined
3193
+ // A `field(…, { leadingTrivia: true })` separator reads the pre-scan cursor.
3194
+ const sepLeads = sepIp >= 0 && leadingTriviaFieldAt(code, k, sepIp)
3103
3195
  const flags = code[ip + 5]!
3104
3196
  const keepSeparators = (flags & 2) !== 0
3105
3197
  const trailingAllowed = (flags & 1) !== 0
@@ -3183,6 +3275,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
3183
3275
  let sp = cur
3184
3276
  if (hasTrivia) sp = skipTrivia(input, sp, ctx)
3185
3277
  ctx._fc = false
3278
+ if (sepLeads) LEAD.at = cur
3186
3279
  const sv = sep(input, sp, ctx)
3187
3280
  if (sv === FAIL) {
3188
3281
  if (needMark) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
@@ -3283,6 +3376,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
3283
3376
  let sp = cur
3284
3377
  if (hasTrivia) sp = skipTrivia(input, sp, ctx)
3285
3378
  ctx._fc = false
3379
+ if (sepLeads) LEAD.at = cur
3286
3380
  const sv = sep(input, sp, ctx)
3287
3381
  if (sv === FAIL) {
3288
3382
  if (needMark) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
@@ -3706,20 +3800,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
3706
3800
  if (child >= 0) scalarFor(child)
3707
3801
  continue
3708
3802
  }
3709
- if (child >= 0) {
3710
- const spec = k[code[child + 1]!]
3711
- if (code[child] === OP_LIT && typeof spec === 'string' && spec.length >= 2) {
3712
- scalarFor(child)
3713
- continue
3714
- }
3715
- }
3716
- const family = leadingLiteralFamily(code, k, armIp)
3717
- if (family !== undefined && family.length >= 2 && family.every(terminal => {
3718
- const value = k[code[terminal + 1]!]
3719
- return typeof value === 'string' && value.length >= 2
3720
- })) {
3721
- for (const terminal of family) scalarFor(terminal)
3722
- }
3803
+ const literals = largeChoiceLiteralPretest(code, k, armIp)
3804
+ if (literals !== undefined) for (const terminal of literals) scalarFor(terminal)
3805
+ if (literals !== undefined && literals.length === 1 && literals[0] === child) continue
3723
3806
  const sequence = leadingScalarSequence(code, armIp)
3724
3807
  if (sequence !== undefined && (sequence.trivia < 0
3725
3808
  || triviaScan[sequence.trivia] != null)) {
@@ -3767,6 +3850,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
3767
3850
  REC ? prog.cc.map((_, i) => sentinelFor(i)) : EMPTY_SENTS,
3768
3851
  matchesAt, recoverScan, orSentinel, captureError,
3769
3852
  scalarRecognizers, commitTriviaScan, scanTriviaCompact, t.lex, adjacencyHolds, t.lexPrograms,
3853
+ LEAD, triviaFieldCapture,
3770
3854
  )
3771
3855
  emitReached = new Set(pre.reached)
3772
3856
  }
@@ -3848,6 +3932,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
3848
3932
  REC ? prog.cc.map((_, i) => sentinelFor(i)) : EMPTY_SENTS,
3849
3933
  matchesAt, recoverScan, orSentinel, captureError,
3850
3934
  scalarRecognizers, commitTriviaScan, scanTriviaCompact, t.lex, adjacencyHolds, t.lexPrograms,
3935
+ LEAD, triviaFieldCapture,
3851
3936
  )
3852
3937
  emitReached = em.reached
3853
3938
  } catch (e) {
@@ -4067,6 +4152,13 @@ export class AssemblyCache {
4067
4152
  }
4068
4153
  }
4069
4154
 
4155
+ /**
4156
+ * One rule of the map. A lowered terminal is emitted as this call, not as
4157
+ * `tableRules(…)["Entry"]`: `@__PURE__` covers a call and not a property read
4158
+ * after it, so a bundler had to keep the whole table of every terminal nothing
4159
+ * referenced. With the lookup inside the call the statement is droppable.
4160
+ */
4161
+ export function tableRules(source: TableProgram | CompactProgram, entry: string): TableRule
4070
4162
  /**
4071
4163
  * The ASSEMBLED rule map — the same artifact contract as `tableRules`, run
4072
4164
  * through linked closures instead of the bytecode interpreter.
@@ -4076,11 +4168,19 @@ export class AssemblyCache {
4076
4168
  * assembly built for it. Everything past that point is pieces, and no piece body
4077
4169
  * reads an option — `scripts/check-invariants.mjs` INV-6 asserts it. `forCtx`
4078
4170
  * carries the argument for why that read is irreducible rather than merely cheap.
4171
+ *
4172
+ * Declared last, so `ReturnType<typeof tableRules>` stays the map.
4079
4173
  */
4080
4174
  export function tableRules(
4081
4175
  source: TableProgram | CompactProgram,
4082
- artifactMetadata: Readonly<Record<symbol, unknown>> = {},
4083
- ): Record<string, TableRule> {
4176
+ artifactMetadata?: Readonly<Record<symbol, unknown>>,
4177
+ ): Record<string, TableRule>
4178
+ export function tableRules(
4179
+ source: TableProgram | CompactProgram,
4180
+ metadataOrEntry: Readonly<Record<symbol, unknown>> | string = {},
4181
+ ): Record<string, TableRule> | TableRule {
4182
+ const entry = typeof metadataOrEntry === 'string' ? metadataOrEntry : undefined
4183
+ const artifactMetadata = typeof metadataOrEntry === 'string' ? {} : metadataOrEntry
4084
4184
  const prog = expandCompact(source)
4085
4185
  const resolved = resolveTable(prog)
4086
4186
  const cache = new AssemblyCache(prog, resolved)
@@ -4105,7 +4205,7 @@ export function tableRules(
4105
4205
  * `runRule`, after the outer read.
4106
4206
  */
4107
4207
  let selected: Assembly | undefined
4108
- return stampRuleMap(prog, {
4208
+ const map = stampRuleMap(prog, {
4109
4209
  runRule: (ri, input, pos, ctx) => {
4110
4210
  const a = selected ?? cache.forCtx(ctx)
4111
4211
  selected = undefined
@@ -4128,4 +4228,8 @@ export function tableRules(
4128
4228
  return a.scanSkip[skipOf?.[ri] ?? -1]
4129
4229
  },
4130
4230
  }, artifactMetadata, resolved)
4231
+ if (entry === undefined) return map
4232
+ // Own properties only: the map inherits `toString`, `constructor` and the rest.
4233
+ if (!Object.hasOwn(map, entry)) throw new Error(`tableRules: the program has no rule ${JSON.stringify(entry)}`)
4234
+ return map[entry]!
4131
4235
  }