sh-ast 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +199 -143
  2. package/dist/analyze/ansi-c-escapes.d.ts +1 -1
  3. package/dist/analyze/ansi-c-escapes.js +1 -1
  4. package/dist/analyze/enumerate-commands.d.ts +11 -16
  5. package/dist/analyze/enumerate-commands.d.ts.map +1 -1
  6. package/dist/analyze/enumerate-commands.js +82 -92
  7. package/dist/analyze/enumerate-commands.js.map +1 -1
  8. package/dist/analyze/index.d.ts +3 -2
  9. package/dist/analyze/index.d.ts.map +1 -1
  10. package/dist/analyze/index.js +1 -1
  11. package/dist/analyze/node-helpers.d.ts +10 -21
  12. package/dist/analyze/node-helpers.d.ts.map +1 -1
  13. package/dist/analyze/node-helpers.js +16 -27
  14. package/dist/analyze/node-helpers.js.map +1 -1
  15. package/dist/analyze/resolve-argv0.d.ts +6 -6
  16. package/dist/analyze/resolve-argv0.js +6 -6
  17. package/dist/analyze/resolve-word.d.ts +7 -10
  18. package/dist/analyze/resolve-word.d.ts.map +1 -1
  19. package/dist/analyze/resolve-word.js +39 -22
  20. package/dist/analyze/resolve-word.js.map +1 -1
  21. package/dist/analyze.d.ts +3780 -397
  22. package/dist/errors.d.ts +6 -79
  23. package/dist/errors.d.ts.map +1 -1
  24. package/dist/errors.js +5 -64
  25. package/dist/errors.js.map +1 -1
  26. package/dist/index.d.ts +20 -14
  27. package/dist/index.d.ts.map +1 -1
  28. package/dist/index.js +26 -4
  29. package/dist/index.js.map +1 -1
  30. package/dist/sh-ast.d.ts +3616 -1020
  31. package/dist/syntax-errors.d.ts +56 -0
  32. package/dist/syntax-errors.d.ts.map +1 -0
  33. package/dist/syntax-errors.js +80 -0
  34. package/dist/syntax-errors.js.map +1 -0
  35. package/dist/syntax-print-errors.d.ts +35 -0
  36. package/dist/syntax-print-errors.d.ts.map +1 -0
  37. package/dist/syntax-print-errors.js +37 -0
  38. package/dist/syntax-print-errors.js.map +1 -0
  39. package/dist/syntax-print.d.ts +49 -0
  40. package/dist/syntax-print.d.ts.map +1 -0
  41. package/dist/syntax-print.js +86 -0
  42. package/dist/syntax-print.js.map +1 -0
  43. package/dist/syntax-snapshot.d.ts +47 -0
  44. package/dist/syntax-snapshot.d.ts.map +1 -0
  45. package/dist/syntax-snapshot.js +121 -0
  46. package/dist/syntax-snapshot.js.map +1 -0
  47. package/dist/syntax-walk-bounds.d.ts +18 -0
  48. package/dist/syntax-walk-bounds.d.ts.map +1 -0
  49. package/dist/syntax-walk-bounds.js +188 -0
  50. package/dist/syntax-walk-bounds.js.map +1 -0
  51. package/dist/syntax-walk-errors.d.ts +19 -0
  52. package/dist/syntax-walk-errors.d.ts.map +1 -0
  53. package/dist/syntax-walk-errors.js +23 -0
  54. package/dist/syntax-walk-errors.js.map +1 -0
  55. package/dist/syntax-walk-license.d.ts +3 -0
  56. package/dist/syntax-walk-license.d.ts.map +1 -0
  57. package/dist/syntax-walk-license.js +37 -0
  58. package/dist/syntax-walk-license.js.map +1 -0
  59. package/dist/syntax-walk.d.ts +46 -0
  60. package/dist/syntax-walk.d.ts.map +1 -0
  61. package/dist/syntax-walk.js +322 -0
  62. package/dist/syntax-walk.js.map +1 -0
  63. package/dist/syntax-wire.d.ts +31 -0
  64. package/dist/syntax-wire.d.ts.map +1 -0
  65. package/dist/syntax-wire.js +202 -0
  66. package/dist/syntax-wire.js.map +1 -0
  67. package/dist/syntax.d.ts +65 -0
  68. package/dist/syntax.d.ts.map +1 -0
  69. package/dist/syntax.js +152 -0
  70. package/dist/syntax.js.map +1 -0
  71. package/dist/wasm-instance.d.ts +5 -8
  72. package/dist/wasm-instance.d.ts.map +1 -1
  73. package/dist/wasm-instance.js +44 -47
  74. package/dist/wasm-instance.js.map +1 -1
  75. package/generated/syntax-constants.d.ts +1652 -0
  76. package/generated/syntax-constants.js +1384 -0
  77. package/generated/syntax-schema.d.ts +60 -0
  78. package/generated/syntax-schema.js +1740 -0
  79. package/generated/syntax-types.d.ts +1977 -0
  80. package/package.json +2 -2
  81. package/shim/sh-ast.wasm +0 -0
  82. package/dist/normalize.d.ts +0 -52
  83. package/dist/normalize.d.ts.map +0 -1
  84. package/dist/normalize.js +0 -260
  85. package/dist/normalize.js.map +0 -1
  86. package/dist/parse.d.ts +0 -50
  87. package/dist/parse.d.ts.map +0 -1
  88. package/dist/parse.js +0 -125
  89. package/dist/parse.js.map +0 -1
  90. package/dist/types.d.ts +0 -74
  91. package/dist/types.d.ts.map +0 -1
  92. package/dist/types.js +0 -2
  93. package/dist/types.js.map +0 -1
  94. package/dist/visitor-keys.d.ts +0 -21
  95. package/dist/visitor-keys.d.ts.map +0 -1
  96. package/dist/visitor-keys.js +0 -23
  97. package/dist/visitor-keys.js.map +0 -1
  98. package/dist/walk.d.ts +0 -18
  99. package/dist/walk.d.ts.map +0 -1
  100. package/dist/walk.js +0 -40
  101. package/dist/walk.js.map +0 -1
  102. package/generated/child-type-schema.d.ts +0 -14
  103. package/generated/child-type-schema.js +0 -188
  104. package/generated/node-types.d.ts +0 -1002
  105. package/generated/position-fields.d.ts +0 -21
  106. package/generated/position-fields.js +0 -50
  107. package/generated/visitor-keys.d.ts +0 -11
  108. package/generated/visitor-keys.js +0 -50
package/README.md CHANGED
@@ -1,53 +1,181 @@
1
1
  # sh-ast
2
2
 
3
- A synchronous, fully-typed shell AST for JavaScript and TypeScript. `sh-ast` parses shell scripts
4
- (bash, POSIX sh, mksh, zsh, and bats) using [mvdan/sh](https://github.com/mvdan/sh) — the Go
5
- shell parser used by `shfmt` and `shellcheck`-adjacent tooling — via a small Go/WASM shim, and
6
- normalizes the result into a plain-object AST with UTF-16-safe positions that's natural to
7
- consume from JS/TS (linters, formatters, codemods, static analysis, or anything else that needs
8
- to understand shell source structurally).
3
+ A synchronous TypeScript interface to the `syntax` package of
4
+ [mvdan/sh](https://github.com/mvdan/sh), pinned to **v3.13.1**. Parsing returns
5
+ mutable records preserving upstream fields, defaults, nulls, numeric operators,
6
+ byte strings, and token positions. First use initializes WebAssembly synchronously
7
+ in Node.js; no initialization promise is required.
8
+
9
+ Traversal uses ordinary TypeScript arrays with explicit mutation limits. The
10
+ [accepted representation decision](../../design/WALK-MUTATION-DECISION.md) records
11
+ which live edits are supported and why this interface does not model Go storage
12
+ capacity, reslicing, or value-slot addresses.
9
13
 
10
14
  ```ts
11
- import { parseSync } from 'sh-ast';
15
+ import { LangBash, nodeEnd, parseSync } from 'sh-ast';
16
+
17
+ const file = parseSync('echo hi', { Variant: LangBash, KeepComments: true });
18
+ const command = file.Stmts?.[0]?.Cmd;
19
+ if (command?.Type === 'CallExpr') {
20
+ console.log(command.Args?.[0]?.Parts); // upstream WordPart alternatives
21
+ console.log(nodeEnd(command).Offset); // exclusive byte offset
22
+ }
23
+ ```
24
+
25
+ ## Public contract
26
+
27
+ - `parseSync(source, options?)` accepts a well-formed Unicode string or exact
28
+ `Uint8Array` bytes. Invalid UTF-8 reaches upstream and produces its parse error;
29
+ lone JavaScript surrogates are rejected before encoding.
30
+ - `ParseOptions` contains `Filename`, `KeepComments`, `Variant`, `StopAt`, and
31
+ `RecoverErrors`. Defaults match upstream, including `KeepComments: false`.
32
+ `Variant` uses exported constants such as `LangBash` and `LangPOSIX`.
33
+ - `File`, `Node`, and concrete records retain upstream spelling and a `Type`
34
+ discriminator. Every field is present; nil slices/pointers use `null`, while an
35
+ allocated empty slice is `[]`. `Slice`, `Replace`, and `Expansion` are auxiliary
36
+ records, not Nodes. Interface unions also admit `TypedNil<Kind>` with
37
+ `{ Type: 'TypedNil', Kind: 'Word' }` spelling: this preserves a Go interface
38
+ holding a nil pointer of a concrete type, distinct from an ordinary `null`
39
+ interface. A tag is allowed only where that concrete type implements the
40
+ declared interface; concrete pointer fields continue to use `null`.
41
+ - `GoString` is `string | Uint8Array`: valid UTF-8 is text; other Go string bytes
42
+ remain bytes. Consumers must explicitly handle both representations.
43
+ - `SyntaxPos` preserves `Offset`, `Line`, `Col`, `IsValid`, `IsRecovered`, and
44
+ `OffsetOverflow`. Offsets and columns count bytes; lines/columns are one-based.
45
+ Invalid, recovered, and overflow states remain distinct. No UTF-16 projection is
46
+ performed by this package.
47
+ - `nodePos(node)` and `nodeEnd(node)` independently compute upstream bounds from
48
+ the current fields, including after edits. They do not return cached parse-time
49
+ bounds. Incomplete constructed nodes can support one bound but fail the other.
50
+ - `parseWithPositionsSync(source, options?)` parses once and returns `{ File,
51
+ Positions }`. `Positions.get(node)` reads immutable original bounds by node
52
+ identity without additional WASM calls. Editing `File` leaves these observations
53
+ unchanged; copied or foreign nodes have no entry. Use this separate snapshot for
54
+ source-location adapters, and live `nodePos`/`nodeEnd` for edited syntax. A
55
+ captured upstream bounds failure is thrown when its pair is requested; it does
56
+ not discard a successful parse. Parser failures retain the same structured
57
+ error and partial File as `parseSync`.
58
+ - `newPos(offset, line, column)` preserves upstream clamping/overflow semantics.
59
+ Its uint64 inputs and the parser's Go int options accept safe integers or exact
60
+ `bigint`; stored position components and operator values are numbers.
61
+ - Operator families have named numeric constants and a distinct raw uint32 path,
62
+ `rawOperator('RedirOperator', value)`, for unnamed upstream values. Representing
63
+ a value does not promise every upstream operation accepts it.
64
+ - `SYNTAX_SCHEMA` and `SYNTAX_INTERFACES` are deeply frozen field/interface
65
+ metadata. Their order is not upstream Walk order.
66
+ - `walk(node, callback)` calls the callback synchronously on entry and with `null`
67
+ on exit. Return `false` on entry to prune that occurrence, including its exit;
68
+ exit boolean values do not control traversal. Every callback result must be a
69
+ boolean; Promises and other values throw `InputError`. Child reads happen live, and
70
+ nested calls keep independent state. Specialized statement/item comment loops
71
+ visit value copies; ordinary comment lists visit their stored objects. The
72
+ array reference and initial length are captured when each list starts. Future
73
+ existing slots are read live; indices added beyond the captured extent are
74
+ ignored by that active traversal. Replacing the owner's array field does not retarget an active list.
75
+ See the mutation limits below.
76
+ - `printSync(node, options?)` reconstructs the current edited tree and collects a
77
+ fresh upstream printer's output as `GoString`. Supported roots are File, Stmt,
78
+ Word, Assign, Command and WordPart alternatives. Only File gets a guaranteed
79
+ final newline. Printing applies upstream formatting; it does not replay the
80
+ original source or promise source-byte identity.
81
+ - `PrintOptions` exposes `Indent`, `BinaryNextLine`, `SwitchCaseIndent`,
82
+ `SpaceRedirects`, `KeepPadding`, `Minify`, `SingleLine`, and `FunctionNextLine`.
83
+ Indent accepts an exact uint64 as a safe number or bigint, with zero selecting
84
+ tabs. Boolean defaults are false. KeepPadding is upstream's best-effort,
85
+ deprecated option; Minify and SingleLine together return an upstream error.
86
+
87
+ `ParseError` and `LangError` preserve upstream fields, exact `Message` bytes, and
88
+ any partial `File`. Their ordinary `Error.message` is a display string.
89
+ `InputError` reports invalid binding inputs, upstream option constraints, or
90
+ unsupported constructed nodes whose requested bound panics upstream.
91
+ `ShInternalError` reports transport failures or unexpected upstream panics.
92
+ `PrintError` retains ordinary upstream printer failures, and `PrintPanicError`
93
+ retains panics from printing representable inputs. `WalkPanicError` represents
94
+ upstream-equivalent traversal panics; callback exceptions retain their identity.
95
+ All extend `ShAstError`. The conservative structural-depth guard throws
96
+ `ShParseMaxDepthError` before potentially exhausting the WASM stack. It is a
97
+ binding resource constraint and can reject deeply nested quoted or commented text.
98
+ Printing collects complete output; large trees or indentation widths can consume
99
+ substantial memory and time. The parse guard does not bound arbitrary constructed
100
+ printing inputs. No arbitrary writer, partial-output, or reusable-printer protocol
101
+ is exposed.
102
+
103
+ ## Traversal mutation limits
104
+
105
+ Supported edits preserve a node's kind and shape: change syntax fields, replace
106
+ pointer children before their read, replace future existing list slots, append
107
+ beyond an active list's captured extent, or assign a new array to the owner field.
108
+ Changes before a list starts are included in its captured reference and length.
109
+ Pruning and synchronous nested walks retain their control behavior; nested walks
110
+ see the tree at their own entry, while edits to shared objects remain live.
111
+
112
+ Keep every index in an active list's captured extent present. Removing one,
113
+ leaving a hole there, or shrinking below that extent is outside the supported
114
+ contract, including through aliases. Truncation, `delete`, `pop`, `shift`, or a
115
+ length-reducing `splice` can do this; removing appended indices beyond the captured
116
+ extent does not affect the active list. No mutation detector or specific
117
+ error/result is promised for those edits. Assign a new owner array
118
+ instead; the active list keeps its original array and later walks see the new one.
119
+ Length-preserving replacement or reordering, including `sort` and `reverse`,
120
+ follows live slot reads and may revisit an object moved into a later slot. Walk
121
+ does not promise to visit each object only once. An appended object moved into an
122
+ original future slot is visited through that slot.
123
+
124
+ Stored Comment callbacks receive JavaScript objects. Replacing a Comment slot
125
+ leaves a retained callback object unchanged; it does not behave as a pointer to a
126
+ Go value slot. Specialized Stmt/CaseItem/ArrayElem comment callbacks receive
127
+ copies, so editing those callback values does not edit the original comments.
128
+ Ordinary JavaScript aliases remain live; this interface does not clone shared
129
+ arrays or objects. It does not model Go slice capacity, distinct slice headers
130
+ sharing backing stores, or reslicing. These limits preclude unrestricted Go
131
+ storage mutation parity while preserving live field edits and upstream traversal control.
132
+
133
+ ## Editing and printing
12
134
 
13
- const file = parseSync('echo hi');
14
- file.stmts[0].cmd.type; // "CallExpr"
135
+ ```ts
136
+ import { parseSync, printSync, walk } from 'sh-ast';
137
+
138
+ const file = parseSync("printf '%s\\n' 'old 🎉'\n", { KeepComments: true });
139
+ walk(file, (node) => {
140
+ if (node?.Type === 'SglQuoted' && node.Value === 'old 🎉') {
141
+ node.Value = 'new 🎉';
142
+ }
143
+ return true;
144
+ });
145
+ const output = printSync(file);
15
146
  ```
16
147
 
17
- ## Provenance
18
-
19
- `sh-ast` was extracted from [`eslint-sh`](https://github.com/mike-north/eslint-sh) (as
20
- `@eslint-sh/bridge`) at commit
21
- [`bdd55b3`](https://github.com/mike-north/eslint-sh/commit/bdd55b3), where it was built as the
22
- parser core underneath an ESLint plugin for shell scripts. The full commit history predating the
23
- extraction lives in that repository. This package carries no ESLint dependency and exposes the
24
- same public API `@eslint-sh/bridge` did.
25
-
26
- ## Status
27
-
28
- The visitor-keys and child-type-schema tables, plus the `ShNodes.*` TypeScript typings, are
29
- generated from mvdan/sh's `syntax` package struct definitions by `tools/gen-visitor-keys` — see
30
- that tool's README for "The schema table is generated, not hand-written".
31
-
32
- ## Public API
33
-
34
- - `parseSync(text, options?)` — synchronous parse; throws `ShParseError` (with `.line`,
35
- `.column`, `.filename`) on a shell syntax error.
36
- - `ParseOptions` — `{ dialect?, filename? }`; `dialect` is one of `bash | posix | mksh | bats |
37
- zsh`, mapped to mvdan/sh's `LangVariant` by string name.
38
- - `ShNode` / `ShFile` — normalized node shape (`type`, `range`, `loc`, plus lowercased fields
39
- from mvdan/sh's tree).
40
- - `ShNodes` — generated, strongly-typed `ShNode` subtypes for every mvdan/sh node type (e.g.
41
- `ShNodes.ShCallExprNode`, `ShNodes.ShDeclClauseNode`), plus union aliases (`ShAnyNode`,
42
- `ShCommandNode`, `ShWordPartNode`, …). Additive and optional — `ShNode`'s index signature
43
- remains the base contract.
44
- - `visitorKeys` — generated node-type → child-field-name table.
45
- - `walk(node, visit)` — visits every node in a normalized tree; discovers children structurally
46
- rather than depending on `visitorKeys`.
148
+ This edits a field without replacing array storage. Node positions still describe
149
+ their upstream token values; live `nodePos` and `nodeEnd` compute from current
150
+ fields, while a separately captured parse snapshot remains tied to original
151
+ source. Consumers such as ESLint can edit a separate tree and use printed output
152
+ as a fix without mutating their original lint view or traversal cache.
153
+
154
+ `sh-ast/analyze` remains a separate syntactic interpretation layer. Its word,
155
+ command, and wrapper analysis consumes the faithful records; it does not define
156
+ parser behavior or the core tree representation.
157
+
158
+ ## Migration from 0.3
159
+
160
+ Fields now use upstream spelling (`Type`, `Stmts`, `Cmd`, `Value`) and explicit
161
+ null/default values. Replace dialect strings with numeric `Variant` constants,
162
+ request comments explicitly, and handle structured errors and byte positions.
163
+ The root no longer exports normalized `ShNode`/`ShNodes`, `range`/`loc`, or
164
+ `visitorKeys`. The new walker uses entry/null-exit callbacks rather than the former
165
+ structural helper's protocol. Consumers such as ESLint own their source-coordinate
166
+ and traversal views. Whole-file parsing, collected printing and the live walker
167
+ with the mutation limits above form this candidate's scope. Stateful/incremental parser protocols
168
+ and full syntax-package parity are deferred.
169
+
170
+ The [design collection](../../design/ARCHITECTURE.md) and
171
+ [implementation evidence](../../design/IMPLEMENTATION.md) describe the release
172
+ boundaries, concrete choices, and validation status. This package has no ESLint
173
+ runtime dependency. It originated in
174
+ [`eslint-sh`](https://github.com/mike-north/eslint-sh) at commit `bdd55b3`.
47
175
 
48
176
  ## Dialect enforcement
49
177
 
50
- `dialect` maps directly onto mvdan/sh's `LangVariant` (`LangBash`, `LangPOSIX`, `LangMirBSDKorn`,
178
+ `Variant` accepts mvdan/sh's `LangVariant` (`LangBash`, `LangPOSIX`, `LangMirBSDKorn`,
51
179
  `LangBats`, `LangZsh`). mvdan/sh gates most dialect-specific syntax with a hard parse error
52
180
  (`checkLang` in its `syntax/parser.go`) when the requested dialect isn't in that construct's
53
181
  allowed set — but two dialect-specific keywords (`[[` test clauses and `let`) have **no such gate**:
@@ -79,12 +207,12 @@ post-processing pass, which `shim/main.go` does not invoke. `{a,b}` always parse
79
207
 
80
208
  \*\*\* zsh's lexer never tokenizes `@(` as an extended-glob opener at all — deliberately, to avoid
81
209
  ambiguity with zsh's own glob qualifiers like `*(N)`. So `ls @(a|b)` parses without error under
82
- `dialect: 'zsh'`, but produces plain `Lit` word parts rather than an `ExtGlob` node; the
210
+ `Variant: LangZsh`, but produces plain `Lit` word parts rather than an `ExtGlob` node; the
83
211
  `checkLang` gate that rejects extended globs under posix is never reached for zsh.
84
212
 
85
213
  ### `[[ ... ]]` and `let` under posix: accepted by design
86
214
 
87
- `dialect: 'posix'` accepting `[[ a == b ]]` and `let x=1` is **intentional upstream behavior, not
215
+ `Variant: LangPOSIX` accepting `[[ a == b ]]` and `let x=1` is **intentional upstream behavior, not
88
216
  a bridge bug**. POSIX sh has no `[[` test clause or `let` builtin as syntax — so mvdan/sh's parser
89
217
  doesn't special-case those keywords at all outside bash-like/mksh/zsh; it falls through to
90
218
  ordinary word/command parsing, exactly as a real POSIX-mode shell would tokenize `[[` as a command
@@ -94,114 +222,42 @@ either by an explicit `checkLang` gate or, for process substitution, after lexer
94
222
  operator inside `[[ a =~ b ]]` remains ordinary command text because POSIX mode never enters the
95
223
  test-clause parser.
96
224
 
97
- Concretely, under `dialect: 'posix'`:
225
+ Concretely, under `Variant: LangPOSIX`:
98
226
 
99
227
  ```ts
100
- const file = parseSync('[[ a == b ]]', { dialect: 'posix' });
101
- file.stmts[0].cmd.type; // "CallExpr" — a command literally named "[[", not "TestClause"
228
+ import { LangPOSIX, parseSync } from 'sh-ast';
229
+
230
+ const file = parseSync('[[ a == b ]]', { Variant: LangPOSIX });
231
+ file.Stmts?.[0]?.Cmd?.Type; // "CallExpr" — a command literally named "[[", not "TestClause"
102
232
  ```
103
233
 
104
234
  Evidence: mvdan/sh's own test suite (`syntax/filetests_test.go`) asserts this exact shape for
105
- `LangPOSIX`, and `shim/main.go` passes the requested dialect through to mvdan/sh's
106
- `LangVariant.Set` unmodified — the bridge does not special-case or mis-map `[[`.
235
+ `LangPOSIX`, and the syntax bridge passes the requested numeric variant to mvdan/sh's
236
+ `Variant` control — the bridge does not special-case or mis-map `[[`.
107
237
 
108
- ## Regenerating the schema tables
238
+ ## Generated model
109
239
 
110
- After bumping the pinned mvdan/sh version (`shim/go.mod` _and_ `../../tools/gen-visitor-keys/go.mod`
111
- must move together):
112
-
113
- ```sh
114
- pnpm run generate:sh-ast-schema
115
- ```
116
-
117
- This regenerates `generated/visitor-keys.js(.d.ts)`, `generated/child-type-schema.js(.d.ts)`, and
118
- `generated/node-types.d.ts`, then formats them with Prettier. CI
119
- (`.github/workflows/gen-visitor-keys.yml`) reruns the same script and fails the build if the
120
- committed artifacts don't come back byte-identical — the drift gate for future mvdan/sh bumps.
121
- Re-run the kitchen-sink golden test (`test/kitchen-sink.test.ts`) afterward; it fails if the new
122
- mvdan/sh version adds a node type the fixtures don't yet exercise.
240
+ The generator reads the pinned upstream struct definitions. Run
241
+ `pnpm run generate:sh-ast-schema` after changing the matching pins in
242
+ `shim/go.mod` and `../../tools/gen-visitor-keys/go.mod`. Faithful declarations,
243
+ constants, and complete field metadata live in `generated/syntax-*`.
244
+ Independent Go reflection tests, manually derived type assertions, and runtime
245
+ fixtures check fidelity; regeneration alone checks only drift.
123
246
 
124
247
  ## The WASM shim
125
248
 
126
- `shim/main.go` parses with `mvdan.cc/sh/v3/syntax` and encodes the result with
127
- `shim/internal/nodeencode` — a performance fork of `mvdan.cc/sh/v3/syntax/typedjson`'s encode
128
- path (see that package's doc comment for why: typedjson's reflection-based encoder rebuilds an
129
- identical synthetic struct type per node _instance_ rather than once per node _type_, which
130
- dominated warm-parse latency — see "Performance" below) — so interface-typed nodes (`CallExpr`,
131
- `BinaryCmd`, …) carry a `Type` discriminator, exactly as upstream typedjson does (verified
132
- byte-identical in `shim/internal/nodeencode/encode_test.go`). The whole result envelope —
133
- including any error message — is marshaled with Go's `encoding/json`, so JSON-significant bytes
134
- in an error message (e.g. an unclosed-quote message containing `"`) survive the Go→JS boundary
135
- intact.
136
-
137
- The shim exposes a linear-memory ABI — `alloc`/`process`/`free` WASM exports plus the `mem`
138
- memory export — instead of a `syscall/js` global function. `wasm-instance.ts` writes UTF-8 bytes
139
- directly into WASM memory and reads the JSON result back with `TextDecoder`, avoiding
140
- `syscall/js`'s much more expensive per-call JS↔Go value marshaling (see "Performance" below).
141
-
142
- ### Performance
143
-
144
- Local measurement (700-line/~9k-node fixture, warm instance, Apple M-series):
145
-
146
- | Stage | Before (syscall/js + typedjson) | After (linear memory + nodeencode) |
147
- | ------------------------------------- | ------------------------------: | ---------------------------------: |
148
- | End-to-end warm `parseSync` (p50) | ~97 ms | ~29 ms |
149
- | wasm call (parse + encode + transfer) | ~91 ms | ~23.5 ms |
150
- | `JSON.parse` | ~3 ms | ~3 ms |
151
- | `normalize` | ~2.5 ms | ~2.7 ms |
152
- | `sh-ast.wasm` size | ~4.10 MB | ~3.82 MB |
153
-
154
- The dominant fix was **not** the transport switch alone — profiling found `typedjson.Encode`'s
155
- `reflect.StructOf` call (rebuilding an identical synthetic struct type once per node _instance_)
156
- responsible for ~80ms of the ~90ms wasm call; `nodeencode`'s per-_type_ cache cut that to ~14ms.
157
- The linear-memory ABI switch and a hand-rolled envelope wrap (skipping a redundant
158
- `encoding/json.Marshal` re-compaction of the already-encoded AST bytes) account for the rest.
159
-
160
- **TinyGo**, evaluated for its ~4x smaller runtime, cannot run this shim: it compiles
161
- `reflect.StructOf` calls without error but panics at runtime with `unimplemented:
162
- reflect.StructOf()` the moment `nodeencode`/`typedjson`'s encoder runs (confirmed against
163
- TinyGo 0.41.1). Standard Go + `wasm-opt -Oz` (binaryen, pinned in CI) is the fallback:
164
- `sh-ast.wasm` is optimized after every build, for a ~7% size reduction (~4.09 MB → ~3.82 MB) with
165
- no behavior change. Neither path reaches a 1.5 MB stretch target.
166
-
167
- After editing `shim/main.go` or `shim/internal/nodeencode`, rebuild the committed
168
- `shim/sh-ast.wasm`:
169
-
170
- ```sh
171
- cd shim
172
- GOOS=js GOARCH=wasm go build -buildvcs=false -trimpath -ldflags="-s -w -buildid=" -o sh-ast.wasm .
173
- wasm-opt -Oz --enable-bulk-memory-opt --enable-nontrapping-float-to-int sh-ast.wasm -o sh-ast.wasm
174
- ```
175
-
176
- `wasm-opt` ships with [binaryen](https://github.com/WebAssembly/binaryen/releases) — CI installs
177
- a pinned release (see `.github/workflows/wasm-reproducibility.yml`'s `BINARYEN_VERSION`); install
178
- the same version locally (`brew install binaryen`, or download the pinned release directly) so
179
- your rebuild matches CI byte-for-byte. `--enable-bulk-memory-opt` and
180
- `--enable-nontrapping-float-to-int` tell `wasm-opt` to accept the two WASM proposal features
181
- Go's `js/wasm` target already emits (both broadly supported in Node and browsers since ~2020) —
182
- without them, `wasm-opt` rejects the input as invalid.
183
-
184
- The `-buildid=` flag blanks out Go's build-ID metadata (used only by the Go toolchain's own
185
- build cache, never read at runtime); without it, `-trimpath` alone is not enough for a
186
- byte-identical rebuild — the embedded build ID otherwise varies between build-cache states even
187
- with identical source and dependencies.
188
-
189
- `-buildvcs=false` is equally required: Go 1.18+ stamps binaries with `vcs.revision`/`vcs.time`/
190
- `vcs.modified` by default (`-buildvcs=auto`), reading whatever commit and dirty-state the
191
- checkout happens to be in at build time. That stamp isn't touched by `-s -w` (it lives outside
192
- the symbol table/DWARF that those flags strip), so two builds of _identical source_ differ
193
- whenever the checkout's commit or dirty-state differs — which is exactly what happens for a
194
- `pull_request`-triggered CI run (GitHub synthesizes a fresh merge commit, with its own SHA and
195
- timestamp, on every run) versus a `push`-triggered run (checks out the named branch commit
196
- as-is), and equally for a local checkout with any uncommitted changes elsewhere in the working
197
- tree. `-buildvcs=false` removes this source of non-determinism entirely; verified byte-for-byte
198
- identical across a native macOS/arm64 build, a `linux/amd64` build under Docker/QEMU emulation,
199
- and real `ubuntu-latest` CI runs triggered via `push`, `pull_request`, and `workflow_dispatch`.
200
-
201
- `wasm-opt -Oz` is a deterministic, version-pinned transform: rebuilding twice (Go build +
202
- wasm-opt) produces byte-identical output, verified locally and by CI's `rebuild-match` job.
203
-
204
- CI rebuilds the shim with a pinned Go toolchain (`shim/go.mod`) and a pinned `wasm-opt`
205
- (`.github/workflows/wasm-reproducibility.yml`), and fails if the output differs from the
206
- committed artifact. It also runs `shim/internal/nodeencode`'s Go test suite, which asserts the
207
- fork's output is byte-identical to upstream `typedjson.Encode`.
249
+ The `syntaxProcess` linear-memory ABI transports exact byte payloads and a
250
+ reversible record encoding. `shim/internal/syntaxwire` preserves all exported
251
+ fields, nil/default values, arbitrary strings, and observable position behavior.
252
+ `shim/internal/syntaxbridge` invokes upstream parsing and position operations.
253
+ Private Go printing tests verify that reconstructed edited trees retain the
254
+ information needed for the future public printer.
255
+
256
+ The canonical binary is built by Linux/amd64 CI with the Go toolchain pinned in
257
+ `shim/go.mod` and the checksum-pinned Binaryen version in
258
+ `.github/workflows/wasm-reproducibility.yml`. Go build flags are
259
+ `-buildvcs=false -trimpath -ldflags="-s -w -buildid="`; optimization uses
260
+ `wasm-opt -Oz --enable-bulk-memory-opt --enable-nontrapping-float-to-int`.
261
+ Use local builds for experiments. Commit the CI-produced artifact, then require
262
+ CI to compare a fresh rebuild byte-for-byte with the committed binary. Historical
263
+ normalized-parser benchmarks do not describe this interface's performance.
@@ -2,7 +2,7 @@
2
2
  * Decodes `$'...'` (Bash ANSI-C Quoting) escape sequences.
3
3
  *
4
4
  * mvdan/sh's parser does **not** perform this decoding: for a `SglQuoted`
5
- * word part with `dollar: true`, the normalized `value` field is the raw,
5
+ * word part with `Dollar: true`, the upstream `Value` field is the raw,
6
6
  * un-decoded source text between the quotes (verified empirically against
7
7
  * mvdan.cc/sh/v3 v3.13.1 — see `test/analyze-resolve-word.test.ts`'s module
8
8
  * comment for how this was confirmed). This module fills that gap using the
@@ -2,7 +2,7 @@
2
2
  * Decodes `$'...'` (Bash ANSI-C Quoting) escape sequences.
3
3
  *
4
4
  * mvdan/sh's parser does **not** perform this decoding: for a `SglQuoted`
5
- * word part with `dollar: true`, the normalized `value` field is the raw,
5
+ * word part with `Dollar: true`, the upstream `Value` field is the raw,
6
6
  * un-decoded source text between the quotes (verified empirically against
7
7
  * mvdan.cc/sh/v3 v3.13.1 — see `test/analyze-resolve-word.test.ts`'s module
8
8
  * comment for how this was confirmed). This module fills that gap using the
@@ -1,4 +1,4 @@
1
- import type { ShNode } from '../types.js';
1
+ import type { CallExpr, GoString, Node } from '../../generated/syntax-types.js';
2
2
  import type { WordResolution } from './resolve-word.js';
3
3
  /**
4
4
  * A single frame of the path from the root of the tree down to a
@@ -19,10 +19,10 @@ import type { WordResolution } from './resolve-word.js';
19
19
  * `ProcSubst` (`<(...)`/`>(...)`) reached from *any* word-bearing
20
20
  * position (an argument, a redirection target, a case subject, a loop's
21
21
  * word list, an assignment value, a test/arithmetic operand, …) — not
22
- * only from `CallExpr.args`.
22
+ * only from `CallExpr.Args`.
23
23
  * - `'if'` (`branch: 'cond' | 'then' | 'else'`) — inside an `IfClause`'s
24
24
  * condition, then-branch, or else-branch (mvdan/sh nests `elif` chains as
25
- * `IfClause.else` pointing to another `IfClause`, so an `elif`'s own
25
+ * `IfClause.Else` pointing to another `IfClause`, so an `elif`'s own
26
26
  * condition/then are reached through an `{kind:'if',branch:'else'}` frame
27
27
  * first, then their own `'cond'`/`'then'` frame — reflecting the real
28
28
  * nesting rather than collapsing it).
@@ -80,7 +80,7 @@ export type CommandContext = {
80
80
  readonly role: 'body' | 'cond';
81
81
  } | {
82
82
  readonly kind: 'function';
83
- readonly name: string;
83
+ readonly name: GoString;
84
84
  } | {
85
85
  readonly kind: 'background';
86
86
  } | {
@@ -100,7 +100,7 @@ export type CommandContext = {
100
100
  */
101
101
  export interface CommandSite {
102
102
  /** The `CallExpr` node this site was found at. */
103
- readonly node: ShNode;
103
+ readonly node: CallExpr;
104
104
  /**
105
105
  * `resolveWord` applied to the first word (`argv[0]`), with
106
106
  * `{ context: 'command-argument' }` — every `CallExpr` word is an
@@ -125,14 +125,14 @@ export interface CommandSite {
125
125
  * `CmdSubst`/`ProcSubst` nested inside words, wherever those words occur
126
126
  * (arguments, redirection targets, case subjects/patterns, loop word
127
127
  * lists, assignment values, test/arithmetic operands, …), not only inside
128
- * `CallExpr.args`.
128
+ * `CallExpr.Args`.
129
129
  *
130
130
  * An assignment-only `CallExpr` (`FOO=bar`, `args` empty) has no first word
131
131
  * to resolve and is not itself a command invocation — no program runs — so
132
132
  * it produces no {@link CommandSite}; any command substitution nested in
133
133
  * its assigned value (`FOO=$(sub)`) is still found and reported.
134
134
  *
135
- * Results are sorted by source position (`node.range[0]`), so nested
135
+ * Results are sorted by source position (`nodePos(node).Offset`), so nested
136
136
  * command substitutions and out-of-structural-order redirection targets
137
137
  * still come back in source order regardless of traversal order.
138
138
  *
@@ -144,14 +144,9 @@ export interface CommandSite {
144
144
  * traversed iteratively and never risks a stack overflow or trips the
145
145
  * nesting-depth guard below, regardless of how many stages/links it has.
146
146
  *
147
- * @param root - Any `ShNode` — typically a `File` from `parseSync`, but any
148
- * subtree (a `Stmt`, a `Command`, or even a bare `Word`) is handled. Note
149
- * `parseSync` itself already refuses (via its own `ShParseMaxDepthError`) any
150
- * input whose estimated nesting exceeds *its* limit before ever producing a
151
- * tree — see `parse-depth-guard.ts` — so a `root` sourced from `parseSync`
152
- * has already passed that earlier, lower-level line of defense; this
153
- * module's own guard below remains a second, independent backstop for
154
- * trees built or mutated some other way.
147
+ * @param root - Any `Node` — typically a `File` from `parseSync`, but any
148
+ * subtree (a `Stmt`, a `Command`, or even a bare `Word`) is handled.
149
+ * The depth guard also applies to caller-built or mutated trees.
155
150
  * @throws {@link ShAnalyzeMaxDepthError} if `root`'s genuinely nested
156
151
  * structure (subshells within subshells, chained command/process
157
152
  * substitutions, deeply nested `if`/`case`/loop/function/`time`/`{ }`
@@ -161,5 +156,5 @@ export interface CommandSite {
161
156
  * it as `deny`.
162
157
  * @public
163
158
  */
164
- export declare function enumerateCommands(root: ShNode): CommandSite[];
159
+ export declare function enumerateCommands(root: Node): CommandSite[];
165
160
  //# sourceMappingURL=enumerate-commands.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"enumerate-commands.d.ts","sourceRoot":"","sources":["../../src/analyze/enumerate-commands.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC/C;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAA;CAAE,GAClE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GACzD;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;CAAE,GAC/B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAA;CAAE,GAC5B;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC;AAEhC;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IAC1B,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;IAC/B,oIAAoI;IACpI,QAAQ,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,CAAC;IACzC,uEAAuE;IACvE,QAAQ,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,CAAC;CAC7C;AA2aD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CAU7D"}
1
+ {"version":3,"file":"enumerate-commands.d.ts","sourceRoot":"","sources":["../../src/analyze/enumerate-commands.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,QAAQ,EACR,QAAQ,EAER,IAAI,EAEL,MAAM,iCAAiC,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC/C;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAA;CAAE,GAClE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GACzD;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,GACtD;IAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;CAAE,GAC/B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAA;CAAE,GAC5B;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC;AAEhC;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IAC1B,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;IAC/B,oIAAoI;IACpI,QAAQ,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,CAAC;IACzC,uEAAuE;IACvE,QAAQ,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,CAAC;CAC7C;AAiaD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,EAAE,CAa3D"}