ucn 5.4.2 → 5.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/.claude/skills/ucn/SKILL.md +58 -236
  2. package/.claude/skills/ucn/references/commands.md +144 -68
  3. package/.claude/skills/ucn/references/resolution-rules.md +171 -0
  4. package/.claude/skills/ucn/references/trust-contract.md +27 -33
  5. package/README.md +27 -16
  6. package/cli/index.js +45 -14
  7. package/core/accessors.js +118 -3
  8. package/core/account.js +62 -11
  9. package/core/analysis.js +517 -266
  10. package/core/ast-analysis.js +76 -117
  11. package/core/async-producers.js +1729 -0
  12. package/core/bridge.js +1877 -268
  13. package/core/brief.js +6 -0
  14. package/core/build-worker.js +78 -14
  15. package/core/cache.js +529 -31
  16. package/core/callers.js +12228 -921
  17. package/core/check.js +30 -38
  18. package/core/class-identity.js +1028 -0
  19. package/core/command-contracts.js +6 -6
  20. package/core/configuration-items.js +86 -0
  21. package/core/contract-membership.js +2262 -0
  22. package/core/cpp-scope.js +532 -0
  23. package/core/deadcode.js +171 -49
  24. package/core/decorator-binding.js +260 -0
  25. package/core/discovery.js +281 -164
  26. package/core/errors.js +52 -0
  27. package/core/execute.js +21 -9
  28. package/core/external-macros.js +326 -0
  29. package/core/graph-build.js +277 -8
  30. package/core/graph.js +35 -2
  31. package/core/ignore-rules.js +472 -0
  32. package/core/imports.js +504 -42
  33. package/core/index-ir.js +94 -11
  34. package/core/inprocess-clients.js +636 -0
  35. package/core/ir.js +87 -8
  36. package/core/macro-expansion.js +1416 -0
  37. package/core/output/analysis.js +18 -2
  38. package/core/output/brief.js +3 -0
  39. package/core/output/doctor.js +4 -0
  40. package/core/output/endpoints.js +15 -6
  41. package/core/output/find.js +7 -2
  42. package/core/output/refactoring.js +121 -14
  43. package/core/output/reporting.js +31 -3
  44. package/core/output/shared.js +14 -0
  45. package/core/output-budget.js +305 -122
  46. package/core/parallel-build.js +124 -47
  47. package/core/parser.js +3 -2
  48. package/core/project.js +472 -66
  49. package/core/provenance-facts.js +156 -12
  50. package/core/provenance-overload.js +30 -3
  51. package/core/provenance.js +104 -30
  52. package/core/pytest-fixtures.js +434 -0
  53. package/core/python-fixture-flow.js +17 -3
  54. package/core/python-modules.js +173 -0
  55. package/core/reflection.js +877 -0
  56. package/core/registry.js +1 -1
  57. package/core/reporting.js +234 -52
  58. package/core/request-helpers.js +561 -0
  59. package/core/route-graph.js +1922 -0
  60. package/core/route-lists.js +2414 -0
  61. package/core/rust-macro-expansion.js +3026 -0
  62. package/core/rust-modules.js +203 -0
  63. package/core/search.js +121 -29
  64. package/core/shared.js +19 -1
  65. package/core/test-name-bindings.js +256 -0
  66. package/core/type-denotation.js +1185 -0
  67. package/core/type-references.js +3229 -0
  68. package/core/verify.js +3360 -672
  69. package/languages/adapter.js +31 -5
  70. package/languages/c-family.js +4061 -212
  71. package/languages/c-macro-invocations.js +380 -0
  72. package/languages/c-preprocessor.js +254 -0
  73. package/languages/csharp.js +1406 -49
  74. package/languages/go.js +1243 -70
  75. package/languages/index.js +728 -6
  76. package/languages/java.js +473 -61
  77. package/languages/javascript.js +1270 -197
  78. package/languages/lexical-scope.js +1280 -0
  79. package/languages/python.js +1557 -72
  80. package/languages/rust-macro-rules.js +1020 -0
  81. package/languages/rust-value-flow.js +416 -0
  82. package/languages/rust.js +2265 -112
  83. package/languages/utils.js +298 -4
  84. package/mcp/server.js +22 -8
  85. package/package.json +1 -1
@@ -0,0 +1,3026 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * Project-local Rust `macro_rules!` expansion (fix #374).
5
+ *
6
+ * A project macro invocation generates code the AST never shows: impl blocks
7
+ * (`delegate_iterator! { Iter<'a, K, V> => .., impl<..> }`), test functions
8
+ * (`quickcheck! { fn prop(a: Iter<u16>) -> bool { .. } }`), forwarding
9
+ * methods (`deref_forward_buf!();` inside an impl) and calls whose receivers
10
+ * only get a type once the arguments sit in their expanded context. This
11
+ * module expands such invocations at build time and makes what they generate
12
+ * ordinary index facts:
13
+ *
14
+ * 1. Invocations are the grammar's `macro_invocation` nodes whose macro
15
+ * resolves to a project `macro_rules!` by Rust's own scoping: textual
16
+ * scope (earlier in the file, `#[macro_use] mod` children, the parent
17
+ * module up to the `mod` declaration), then path scope
18
+ * (`#[macro_export]` macros through `crate::`/`$crate::`/package paths,
19
+ * `use` bindings, globs, `#[macro_use] extern crate`).
20
+ * 2. languages/rust-macro-rules.js matches the invocation's token trees
21
+ * against the rules and transcribes; project macros invoked by the
22
+ * transcription expand recursively (bounded depth and output), locals
23
+ * the transcriber introduces are renamed apart (hygiene) and `$crate`
24
+ * names the defining crate.
25
+ * 3. The file is re-analyzed with every expansion spliced in place of its
26
+ * invocation, laid out on the invocation's own lines (argument tokens
27
+ * keep their line), so the full Rust parser extracts functions, impls,
28
+ * calls and receiver facts in their real context. The file's symbols
29
+ * and call records become those of the expanded file: facts produced
30
+ * inside an invocation carry `macroExpansion {macro, definition, origin,
31
+ * line}` (origin 'argument' when the token was written in the
32
+ * invocation, 'template' when it came from the transcriber).
33
+ *
34
+ * An invocation whose rules do not match, whose expansion exceeds the bounds
35
+ * or does not parse in its context stays as written (the parser's token-tree
36
+ * view) and is recorded as blind, disclosed by `repo --sections=health
37
+ * --deep` and deadcode. Macros from dependencies (`vec!`, `println!`, a
38
+ * crate's `quickcheck!`) are never expanded: their definitions are not in the
39
+ * project. Results are persisted with the file entry and revalidated against
40
+ * the file, every macro definition it used and its module ancestry.
41
+ */
42
+
43
+ const crypto = require('crypto');
44
+ const fs = require('fs');
45
+ const path = require('path');
46
+ const { getParser, safeParse, PARSE_OPTIONS } = require('../languages');
47
+ const { addIRSymbol } = require('./index-ir');
48
+ const { findCargoRoot } = require('./imports');
49
+ const {
50
+ MacroBlind, argTokensOf, compileMacro, expandWithRules, isPostfixChain, tokenCount,
51
+ } = require('../languages/rust-macro-rules');
52
+ const { codeUnitCompare } = require('./shared');
53
+
54
+ const MAX_DEPTH = 64;
55
+ const MAX_OUTPUT_TOKENS = 20000;
56
+ const BLIND_SAMPLE = 20;
57
+
58
+ function isRust(fileEntry) {
59
+ return fileEntry?.language === 'rust';
60
+ }
61
+
62
+ /** name -> Rust macro_rules! definitions. */
63
+ function macroInventory(index) {
64
+ const inventory = new Map();
65
+ for (const [name, defs] of index.symbols) {
66
+ for (const def of defs) {
67
+ if (def.type !== 'macro') continue;
68
+ if (!isRust(index.files.get(def.file))) continue;
69
+ if (!inventory.has(name)) inventory.set(name, []);
70
+ inventory.get(name).push(def);
71
+ }
72
+ }
73
+ return inventory;
74
+ }
75
+
76
+ function defKey(def) {
77
+ return `${def.relativePath}:${def.startLine}`;
78
+ }
79
+
80
+ // ── Per-pass state ────────────────────────────────────────────────────────
81
+
82
+ function createState(index, inventory = null) {
83
+ return {
84
+ index,
85
+ parser: getParser('rust'),
86
+ inventory: inventory || macroInventory(index),
87
+ contents: new Map(),
88
+ trees: new Map(),
89
+ compiled: new Map(),
90
+ parents: null,
91
+ packages: new Map(), // file dir -> cargo info
92
+ crateNames: null, // crate name -> package root dir
93
+ roots: new Map(), // file -> crate root file
94
+ macroUseCrates: new Map(), // crate root file -> Set<crate name>
95
+ lineStarts: new Map(),
96
+ textualMemo: new Map(),
97
+ productive: new Map(),
98
+ hygieneSeq: 0,
99
+ contexts: new Map(), // file -> calls-pass context (fix #405)
100
+ resolved: new Map(), // invocation -> resolveMacro answer
101
+ };
102
+ }
103
+
104
+ function readContent(state, file) {
105
+ let content = state.contents.get(file);
106
+ if (content === undefined) {
107
+ try {
108
+ content = state.index._readFile(file);
109
+ } catch {
110
+ content = null;
111
+ }
112
+ state.contents.set(file, content);
113
+ }
114
+ return content;
115
+ }
116
+
117
+ function treeOf(state, file) {
118
+ if (state.trees.has(file)) return state.trees.get(file);
119
+ const content = readContent(state, file);
120
+ const tree = content === null ? null : safeParse(state.parser, content, undefined, PARSE_OPTIONS);
121
+ state.trees.set(file, tree);
122
+ return tree;
123
+ }
124
+
125
+ /** child module file -> { file, line, macroUse } from `mod x;` declarations. */
126
+ function moduleParents(state) {
127
+ if (state.parents) return state.parents;
128
+ const parents = new Map();
129
+ const index = state.index;
130
+ for (const [file, entry] of index.files) {
131
+ if (!isRust(entry) || !entry.moduleResolved) continue;
132
+ for (const detail of entry.importDetails || []) {
133
+ if (detail.type !== 'mod') continue;
134
+ const rel = entry.moduleResolved[detail.module];
135
+ if (!rel) continue;
136
+ const child = path.join(index.root, rel);
137
+ if (child === file || parents.has(child)) continue;
138
+ const decl = (entry.symbols || []).find(symbol => symbol.type === 'module' &&
139
+ symbol.name === detail.module && symbol.startLine === detail.line);
140
+ parents.set(child, {
141
+ file, line: detail.line,
142
+ macroUse: !!decl?.modifiers?.includes('macro_use'),
143
+ });
144
+ }
145
+ }
146
+ state.parents = parents;
147
+ return parents;
148
+ }
149
+
150
+ function crateRoot(state, file) {
151
+ if (state.roots.has(file)) return state.roots.get(file);
152
+ const parents = moduleParents(state);
153
+ let current = file;
154
+ const seen = new Set();
155
+ while (parents.has(current) && !seen.has(current)) {
156
+ seen.add(current);
157
+ current = parents.get(current).file;
158
+ }
159
+ state.roots.set(file, current);
160
+ return current;
161
+ }
162
+
163
+ function packageOf(state, file) {
164
+ const dir = path.dirname(file);
165
+ if (!state.packages.has(dir)) state.packages.set(dir, findCargoRoot(dir) || null);
166
+ return state.packages.get(dir);
167
+ }
168
+
169
+ function packageByCrateName(state, name) {
170
+ if (!state.crateNames) {
171
+ state.crateNames = new Map();
172
+ for (const [file, entry] of state.index.files) {
173
+ if (!isRust(entry)) continue;
174
+ const info = packageOf(state, file);
175
+ if (info?.packageName && !state.crateNames.has(info.packageName)) {
176
+ state.crateNames.set(info.packageName, info.root);
177
+ }
178
+ }
179
+ }
180
+ return state.crateNames.get(name) || null;
181
+ }
182
+
183
+ function isLibRoot(state, rootFile) {
184
+ const info = packageOf(state, rootFile);
185
+ return !!info && rootFile === path.join(info.srcDir, 'lib.rs');
186
+ }
187
+
188
+ /** Tokens `$crate` becomes for a definition expanded at `siteFile`. */
189
+ function crateTokensFor(state, def, siteFile) {
190
+ const defRoot = crateRoot(state, def.file);
191
+ if (defRoot === crateRoot(state, siteFile) || !isLibRoot(state, defRoot)) {
192
+ return [{ k: 'ident', v: 'crate' }];
193
+ }
194
+ const name = packageOf(state, def.file)?.packageName;
195
+ return [{ k: 'ident', v: name || 'crate' }];
196
+ }
197
+
198
+ // ── Definitions ───────────────────────────────────────────────────────────
199
+
200
+ function attributeNames(node) {
201
+ const names = [];
202
+ for (let sibling = node.previousNamedSibling; sibling; sibling = sibling.previousNamedSibling) {
203
+ if (sibling.type === 'line_comment' || sibling.type === 'block_comment') continue;
204
+ if (sibling.type !== 'attribute_item') break;
205
+ const attribute = sibling.namedChildren.find(child => child.type === 'attribute');
206
+ const head = attribute?.namedChildren.find(child =>
207
+ child.type === 'identifier' || child.type === 'scoped_identifier');
208
+ if (head) names.push(head.text);
209
+ }
210
+ return names;
211
+ }
212
+
213
+ /** Compiled definition + lexical scope, memoized per pass. */
214
+ function compiledDef(state, def) {
215
+ const key = defKey(def);
216
+ if (state.compiled.has(key)) return state.compiled.get(key);
217
+ // Attributes and textual scope are parser facts of the definition; the
218
+ // rules are compiled from the definition's own text, parsed alone with
219
+ // its lines kept (no re-parse of the defining file).
220
+ const attrs = def.modifiers || [];
221
+ const start = lineOffset(state, def.file, def.startLine);
222
+ const end = lineOffset(state, def.file, def.endLine + 1);
223
+ const scope = def.macroScope ? {
224
+ kind: def.macroScope.kind,
225
+ start: lineOffset(state, def.file, def.macroScope.startLine),
226
+ end: lineOffset(state, def.file, def.macroScope.endLine + 1),
227
+ ...(def.macroScope.macroUse && { macroUse: true }),
228
+ } : null;
229
+ const base = { start, end, exported: attrs.includes('macro_export'), cfg: attrs.includes('cfg'), scope };
230
+ let result;
231
+ const content = readContent(state, def.file);
232
+ const text = content === null ? null
233
+ : '\n'.repeat(def.startLine - 1) + content.slice(start, end);
234
+ const tree = text === null ? null : safeParse(state.parser, text, undefined, PARSE_OPTIONS);
235
+ const node = tree && tree.rootNode.descendantsOfType('macro_definition').find(candidate =>
236
+ candidate.startPosition.row + 1 === def.startLine &&
237
+ candidate.childForFieldName('name')?.text === def.name);
238
+ if (!node) {
239
+ result = { ...base, error: 'definition-unavailable' };
240
+ } else {
241
+ try {
242
+ result = { ...base, compiled: compileMacro(node) };
243
+ } catch (error) {
244
+ if (!(error instanceof MacroBlind)) throw error;
245
+ result = { ...base, error: error.reason };
246
+ }
247
+ }
248
+ state.compiled.set(key, result);
249
+ return result;
250
+ }
251
+
252
+ // ── Resolution ────────────────────────────────────────────────────────────
253
+
254
+ /**
255
+ * Definitions of `name` in textual scope at (file, offset); `offset`
256
+ * Infinity asks what a module exports to its parent through #[macro_use].
257
+ * Returns [{def, pos}] candidates of the innermost scope that has any.
258
+ */
259
+ function textualCandidates(state, file, offset, line, name, walkParents, seen = new Set()) {
260
+ if (seen.has(file)) return [];
261
+ seen.add(file);
262
+ const candidates = [];
263
+ for (const def of state.inventory.get(name) || []) {
264
+ if (def.file !== file) continue;
265
+ const info = compiledDef(state, def);
266
+ if (info.start === undefined) continue;
267
+ if (offset !== Infinity && info.end > offset) continue;
268
+ const scope = info.scope;
269
+ if (scope) {
270
+ const inside = offset >= scope.start && offset <= scope.end;
271
+ const after = scope.kind === 'mod' && scope.macroUse && offset >= scope.end;
272
+ if (offset === Infinity ? !(scope.kind === 'mod' && scope.macroUse) : !(inside || after)) continue;
273
+ }
274
+ candidates.push({ def, pos: info.start, info });
275
+ }
276
+ // `#[macro_use] mod child;` before this point brings the child's macros.
277
+ const entry = state.index.files.get(file);
278
+ const parents = moduleParents(state);
279
+ for (const detail of entry?.importDetails || []) {
280
+ if (detail.type !== 'mod' || !(detail.line < line || line === Infinity)) continue;
281
+ const rel = entry.moduleResolved?.[detail.module];
282
+ if (!rel) continue;
283
+ const child = path.join(state.index.root, rel);
284
+ const link = parents.get(child);
285
+ if (!link || link.file !== file || !link.macroUse) continue;
286
+ const exported = textualCandidates(state, child, Infinity, Infinity, name, false, seen);
287
+ // Position: the declaration line (same-line declarations are not
288
+ // idiomatic), keeping the child's own order among its definitions
289
+ // so its cfg alternatives stay together.
290
+ const declared = lineOffset(state, file, detail.line);
291
+ for (const candidate of exported) {
292
+ candidates.push({ ...candidate, pos: declared + candidate.pos / 1e12 });
293
+ }
294
+ }
295
+ if (candidates.length > 0) return candidates;
296
+ if (!walkParents) return [];
297
+ const parent = parents.get(file);
298
+ if (!parent) return [];
299
+ // What the parent sees at the child's `mod` declaration does not depend
300
+ // on where in the child the invocation is.
301
+ const key = `${parent.file}\0${parent.line}\0${name}`;
302
+ if (!state.textualMemo.has(key)) {
303
+ state.textualMemo.set(key, textualCandidates(state, parent.file,
304
+ lineOffset(state, parent.file, parent.line), parent.line, name, true, new Set(seen)));
305
+ }
306
+ return state.textualMemo.get(key);
307
+ }
308
+
309
+ function lineOffset(state, file, line) {
310
+ let starts = state.lineStarts.get(file);
311
+ if (!starts) {
312
+ const content = readContent(state, file);
313
+ starts = [0];
314
+ if (content !== null) {
315
+ for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) starts.push(i + 1);
316
+ }
317
+ state.lineStarts.set(file, starts);
318
+ }
319
+ return starts[Math.min(Math.max(line - 1, 0), starts.length - 1)];
320
+ }
321
+
322
+ /** Offset just past `line` (its newline included), or the content's end. */
323
+ function lineEndOffset(state, file, line) {
324
+ lineOffset(state, file, 1);
325
+ const starts = state.lineStarts.get(file);
326
+ return line < starts.length ? starts[line] : (readContent(state, file) || '').length;
327
+ }
328
+
329
+ function pickLatest(candidates) {
330
+ return candidates.reduce((best, candidate) => (!best || candidate.pos > best.pos ? candidate : best), null);
331
+ }
332
+
333
+ /**
334
+ * Definitions a bare or path invocation of `name` resolves to at
335
+ * (file, offset). Returns { defs, ambiguous } or null (not a project macro
336
+ * in scope).
337
+ */
338
+ function resolveMacro(state, file, offset, line, name, qualifier, dollarCratePackage) {
339
+ // An invocation's answer is asked again by the pass that expands it
340
+ // (fix #405: fileMayExpand resolved every invocation first).
341
+ const key = `${file}\0${offset}\0${line}\0${name}\0${qualifier || ''}\0${dollarCratePackage || ''}`;
342
+ if (state.resolved.has(key)) return state.resolved.get(key);
343
+ const result = resolveMacroUncached(state, file, offset, line, name, qualifier, dollarCratePackage);
344
+ state.resolved.set(key, result);
345
+ return result;
346
+ }
347
+
348
+ function resolveMacroUncached(state, file, offset, line, name, qualifier, dollarCratePackage) {
349
+ const all = state.inventory.get(name);
350
+ if (!all || all.length === 0) return null;
351
+ const inPackage = pkgRoot => all.filter(def => packageOf(state, def.file)?.root === pkgRoot);
352
+ const preferExported = defs => {
353
+ const exported = defs.filter(def => compiledDef(state, def).exported);
354
+ return exported.length > 0 ? exported : defs;
355
+ };
356
+ const finish = defs => {
357
+ if (defs.length === 0) return null;
358
+ return { defs: defs.slice(0, 4), ambiguous: defs.length > 1 };
359
+ };
360
+ if (qualifier) {
361
+ const first = qualifier.split('::')[0];
362
+ const pkgRoot = dollarCratePackage ||
363
+ (['crate', 'self', 'super', '$crate'].includes(first) ? packageOf(state, file)?.root
364
+ : packageByCrateName(state, first));
365
+ if (!pkgRoot) return null;
366
+ return finish(preferExported(inPackage(pkgRoot)));
367
+ }
368
+ const textual = textualCandidates(state, file, offset, line, name, true);
369
+ if (textual.length > 0) {
370
+ const best = pickLatest(textual);
371
+ // Configuration alternatives: several cfg-gated definitions visible
372
+ // from the same scope.
373
+ const alternatives = best.info?.cfg
374
+ ? textual.filter(candidate => candidate.info?.cfg && candidate.def.file === best.def.file)
375
+ : [best];
376
+ // Every configuration's definition expands: cfg-gated code is code.
377
+ return {
378
+ defs: alternatives.map(candidate => candidate.def).slice(0, 4),
379
+ alternatives: alternatives.length > 1,
380
+ ambiguous: false,
381
+ };
382
+ }
383
+ const own = packageOf(state, file);
384
+ const root = crateRoot(state, file);
385
+ const entry = state.index.files.get(file);
386
+ // #[macro_export] macros live in their crate root's namespace.
387
+ if (own && root === file) {
388
+ const exported = inPackage(own.root).filter(def => compiledDef(state, def).exported &&
389
+ crateRoot(state, def.file) === root);
390
+ if (exported.length > 0) return finish(exported);
391
+ }
392
+ // `use crate::m;` / `use pkg::m;` / `use pkg::path::m as n;`
393
+ for (const binding of entry?.importBindings || []) {
394
+ const local = binding.alias || binding.name;
395
+ if (local !== name) continue;
396
+ const first = String(binding.module || '').split('::')[0];
397
+ const pkgRoot = ['crate', 'self', 'super'].includes(first) ? own?.root
398
+ : packageByCrateName(state, first);
399
+ if (!pkgRoot) continue;
400
+ const found = finish(preferExported(inPackage(pkgRoot)));
401
+ if (found) return found;
402
+ }
403
+ // `use pkg::*;` and `#[macro_use] extern crate pkg;`
404
+ const crates = new Set();
405
+ for (const detail of entry?.importDetails || []) {
406
+ if (detail.type === 'use-glob' && !String(detail.module).includes('::')) crates.add(detail.module);
407
+ }
408
+ for (const crate of macroUseCrates(state, root)) crates.add(crate);
409
+ for (const crate of crates) {
410
+ const pkgRoot = crate === 'crate' ? own?.root : packageByCrateName(state, crate);
411
+ if (!pkgRoot) continue;
412
+ const exported = inPackage(pkgRoot).filter(def => compiledDef(state, def).exported);
413
+ const found = finish(exported);
414
+ if (found) return found;
415
+ }
416
+ return null;
417
+ }
418
+
419
+ /** Crates whose macros a crate root imports with `#[macro_use] extern crate`. */
420
+ function macroUseCrates(state, rootFile) {
421
+ if (state.macroUseCrates.has(rootFile)) return state.macroUseCrates.get(rootFile);
422
+ const crates = new Set();
423
+ const content = readContent(state, rootFile);
424
+ if (content !== null && content.includes('macro_use')) {
425
+ const tree = treeOf(state, rootFile);
426
+ for (const node of tree?.rootNode.namedChildren || []) {
427
+ if (node.type !== 'extern_crate_declaration') continue;
428
+ if (!attributeNames(node).includes('macro_use')) continue;
429
+ const name = node.childForFieldName('name')?.text;
430
+ if (name) crates.add(name);
431
+ }
432
+ }
433
+ state.macroUseCrates.set(rootFile, crates);
434
+ return crates;
435
+ }
436
+
437
+ // ── Productivity ──────────────────────────────────────────────────────────
438
+
439
+ /**
440
+ * Whether expanding a macro can change the facts of the code it expands to
441
+ * beyond the parser's token-tree view of the invocation (whose argument
442
+ * calls are already recorded and typed in scope, and whose template calls
443
+ * are already visible at the definition): it declares items, binds argument
444
+ * fragments as locals or parameters, places an argument in callee position
445
+ * (`$f(..)`, `x.$m(..)`), or invokes a project macro that does.
446
+ * Other macros (logging, `try!`-style matches, assertions) are not expanded.
447
+ */
448
+ /**
449
+ * true (always worth expanding), 'callee' (only where an argument names a
450
+ * project callable) or false.
451
+ */
452
+ function productivity(state, def, visiting = new Set()) {
453
+ const key = defKey(def);
454
+ if (state.productive.has(key)) return state.productive.get(key);
455
+ if (visiting.has(key)) return false;
456
+ visiting.add(key);
457
+ const info = compiledDef(state, def);
458
+ let productive;
459
+ if (info.compiled) {
460
+ productive = info.compiled.declares || info.compiled.bindsArgs ? true
461
+ : info.compiled.callsArgs ? 'callee' : false;
462
+ for (const name of info.compiled.invokes) {
463
+ if (productive === true) break;
464
+ for (const other of state.inventory.get(name) || []) {
465
+ const nested = productivity(state, other, visiting);
466
+ if (nested === true) productive = true;
467
+ else if (nested === 'callee' && !productive) productive = 'callee';
468
+ }
469
+ }
470
+ } else {
471
+ // A definition the engine cannot model is expanded (and disclosed).
472
+ productive = true;
473
+ }
474
+ state.productive.set(key, productive);
475
+ return productive;
476
+ }
477
+
478
+ /**
479
+ * Whether an invocation of `defs` is worth expanding given its source text:
480
+ * a macro that only moves arguments into callee position is expanded where
481
+ * an argument word names a project callable (a word scan of the invocation
482
+ * is only a prefilter; the expansion decides what is a call).
483
+ */
484
+ function worthExpanding(state, defs, text, expressionContext) {
485
+ let calleeOnly = false;
486
+ for (const def of defs) {
487
+ const level = productivity(state, def);
488
+ if (level === true) return true;
489
+ if (level === 'callee') calleeOnly = true;
490
+ }
491
+ if (expressionContext && forwardsValue(state, defs)) return true;
492
+ if (!calleeOnly || !text) return false;
493
+ for (const word of new Set(text.match(/[A-Za-z_][A-Za-z0-9_]*/g) || [])) {
494
+ if (namesCallable(state.index, word)) return true;
495
+ }
496
+ return false;
497
+ }
498
+
499
+ const NON_CALLABLE_DEF_TYPES = new Set(['macro', 'struct', 'enum', 'trait', 'module', 'type', 'impl', 'field']);
500
+
501
+ /** Whether a project definition named `word` can be called. */
502
+ function namesCallable(index, word) {
503
+ if (index.callableNames) return index.callableNames.has(word);
504
+ const defs = index.symbols.get(word);
505
+ return !!defs && defs.some(def => !NON_CALLABLE_DEF_TYPES.has(def.type));
506
+ }
507
+
508
+ /** An expression-position invocation whose value is one of its arguments. */
509
+ function forwardsValue(state, defs) {
510
+ return defs.some(def => compiledDef(state, def).compiled?.forwardsValue);
511
+ }
512
+
513
+ /**
514
+ * Whether any recorded invocation of the file can need expanding, decided
515
+ * from call records before the file is parsed again.
516
+ */
517
+ function fileMayExpand(state, filePath) {
518
+ const calls = state.index.callsCache.get(filePath)?.calls || [];
519
+ let memberRanges = null;
520
+ const inImplOrTrait = line => {
521
+ memberRanges ||= (state.index.files.get(filePath)?.symbols || [])
522
+ .filter(symbol => symbol.type === 'impl' || symbol.type === 'trait')
523
+ .map(symbol => [symbol.startLine, symbol.endLine]);
524
+ return memberRanges.some(([start, end]) => line >= start && line <= end);
525
+ };
526
+ for (const call of calls) {
527
+ if (!call.isMacro || call.inMacro || call.macroExpansion || !state.inventory.has(call.name)) continue;
528
+ if (!Number.isInteger(call.callStart)) return true;
529
+ const qualifier = call.isPathMacro ? call.receiver : null;
530
+ const resolved = resolveMacro(state, filePath, call.callStart, call.line, call.name, qualifier, null);
531
+ if (!resolved) continue;
532
+ // Item wrappers (`cfg_x! { items }`) at module level declare what the
533
+ // parser already recovers from their token trees.
534
+ if (resolved.defs.every(def => compiledDef(state, def).compiled?.passThroughItems) &&
535
+ !call.enclosingFunction && !inImplOrTrait(call.line)) continue;
536
+ const content = readContent(state, filePath);
537
+ const text = content === null ? null : content.slice(call.callStart, call.callEnd || call.callStart);
538
+ // Expression position is a parser fact of the invocation record.
539
+ if (worthExpanding(state, resolved.defs, text, call.macroExpr === true)) return true;
540
+ }
541
+ return false;
542
+ }
543
+
544
+ // ── Expansion ─────────────────────────────────────────────────────────────
545
+
546
+ function invocationName(node) {
547
+ const macroNode = node.childForFieldName('macro');
548
+ if (!macroNode) return null;
549
+ const parts = macroNode.text.split('::').filter(Boolean);
550
+ const name = parts.pop();
551
+ if (!name) return null;
552
+ return { name, qualifier: parts.length > 0 ? parts.join('::') : null };
553
+ }
554
+
555
+ /** Syntactic context the invocation's expansion is parsed in. */
556
+ function invocationContext(node) {
557
+ return require('../languages/rust').rustMacroInvocationContext(node);
558
+ }
559
+
560
+ function itemRecoveryPosition(node) {
561
+ if (node.parent?.type === 'source_file') return true;
562
+ return node.parent?.type === 'declaration_list' && node.parent.parent?.type === 'mod_item';
563
+ }
564
+
565
+ function isTok(tok, v) {
566
+ return tok && tok.k !== 'group' && tok.v === v;
567
+ }
568
+
569
+ /**
570
+ * Expand `toks` (an invocation's token list) with one definition, then every
571
+ * project macro the transcription invokes. `site` carries the budgets, the
572
+ * used definitions and blind records of this top-level invocation.
573
+ */
574
+ function expandTokens(state, site, def, toks, depth) {
575
+ if (depth > MAX_DEPTH) throw new MacroBlind('recursion-depth');
576
+ const info = compiledDef(state, def);
577
+ if (info.error) throw new MacroBlind(info.error);
578
+ site.used.set(defKey(def), def);
579
+ const ctx = {
580
+ macro: def.name,
581
+ hygiene: ++state.hygieneSeq,
582
+ crateTokens: crateTokensFor(state, def, site.file),
583
+ budget: site.budget,
584
+ };
585
+ const { tokens } = expandWithRules(info.compiled, toks, ctx);
586
+ site.budget -= tokenCount(tokens);
587
+ if (site.budget < 0) throw new MacroBlind('output-budget');
588
+ return expandNested(state, site, def, tokens, depth);
589
+ }
590
+
591
+ function expandNested(state, site, def, tokens, depth) {
592
+ const out = [];
593
+ for (let i = 0; i < tokens.length; i++) {
594
+ const tok = tokens[i];
595
+ if (tok.k === 'group') {
596
+ out.push({ ...tok, c: expandNested(state, site, def, tok.c, depth) });
597
+ continue;
598
+ }
599
+ if (tok.k !== 'ident' || isTok(tokens[i - 1], '::') || tok.v === 'macro_rules') {
600
+ out.push(tok);
601
+ continue;
602
+ }
603
+ let k = i;
604
+ const segments = [tok.v];
605
+ while (isTok(tokens[k + 1], '::') && tokens[k + 2]?.k === 'ident') {
606
+ k += 2;
607
+ segments.push(tokens[k].v);
608
+ }
609
+ const group = tokens[k + 2];
610
+ if (!isTok(tokens[k + 1], '!') || group?.k !== 'group') {
611
+ out.push(tok);
612
+ continue;
613
+ }
614
+ const name = segments.pop();
615
+ const qualifier = segments.length > 0 ? segments.join('::') : null;
616
+ const dollarCratePackage = tok.dollarCrate ? packageOf(state, def.file)?.root : null;
617
+ let resolved = resolveMacro(state, site.file, site.offset, site.line, name, qualifier, dollarCratePackage);
618
+ if (!resolved && !qualifier) {
619
+ const info = compiledDef(state, def);
620
+ resolved = resolveMacro(state, def.file, info.start ?? 0, def.startLine, name, null, null);
621
+ }
622
+ if (!resolved) {
623
+ out.push(tok);
624
+ continue;
625
+ }
626
+ const prev = out[out.length - 1];
627
+ const following = tokens[k + 3];
628
+ // Statement/item position: at the start of a token list or after a
629
+ // statement, as a braced invocation or one ending its statement.
630
+ const statementPosition = (!prev || isTok(prev, ';') || (prev.k === 'group' && prev.d === '{')) &&
631
+ (group.d === '{' || !following || isTok(following, ';'));
632
+ let inner;
633
+ try {
634
+ inner = expandAlternatives(state, site, resolved, group.c, depth + 1,
635
+ statementPosition ? 'stmts' : 'expr');
636
+ if (resolved.ambiguous) site.ambiguous = true;
637
+ } catch (error) {
638
+ if (!(error instanceof MacroBlind)) throw error;
639
+ if (error.reason === 'output-budget') throw error;
640
+ site.nestedBlind.push({ macro: name, reason: error.reason });
641
+ out.push(tok);
642
+ continue;
643
+ }
644
+ out.push(...(statementPosition ? inner : wrapExpression(inner)));
645
+ i = k + 2;
646
+ }
647
+ return out;
648
+ }
649
+
650
+ /**
651
+ * Expand with every configuration alternative of a macro. Items and
652
+ * statements of each alternative are all kept (each is real code under its
653
+ * configuration); differing expression values become the arms of one
654
+ * `match`, so no configuration's calls are lost. Throws MacroBlind when no
655
+ * alternative expands.
656
+ */
657
+ function expandAlternatives(state, site, resolved, toks, depth, context) {
658
+ const outputs = [];
659
+ const texts = new Set();
660
+ let failure = null;
661
+ for (const def of resolved.defs) {
662
+ try {
663
+ const tokens = expandTokens(state, site, def, cloneTree(toks), depth);
664
+ if (resolved.defs.length > 1) {
665
+ const text = tokenText(tokens);
666
+ if (texts.has(text)) continue;
667
+ texts.add(text);
668
+ }
669
+ outputs.push(tokens);
670
+ } catch (error) {
671
+ if (!(error instanceof MacroBlind) || error.reason === 'output-budget') throw error;
672
+ failure = failure || error;
673
+ }
674
+ }
675
+ if (outputs.length === 0) throw failure || new MacroBlind('no-matching-rule');
676
+ if (failure) site.nestedBlind.push({ macro: resolved.defs[0].name, reason: `alternative:${failure.reason}` });
677
+ if (outputs.length === 1) return outputs[0];
678
+ if (context !== 'expr') return outputs.flat();
679
+ const arms = [];
680
+ outputs.forEach((tokens, i) => {
681
+ if (i > 0) arms.push({ k: 'punct', v: ',', src: 'tpl' });
682
+ arms.push(i === outputs.length - 1 ? { k: 'ident', v: '_', src: 'tpl' } : { k: 'lit', v: String(i), src: 'tpl' },
683
+ { k: 'punct', v: '=>', src: 'tpl' }, ...wrapExpression(tokens));
684
+ });
685
+ return [{ k: 'group', v: '(', d: '(', src: 'tpl', synthetic: true, c: [
686
+ { k: 'ident', v: 'match', src: 'tpl' }, { k: 'lit', v: '0', src: 'tpl' },
687
+ { k: 'group', v: '{', d: '{', src: 'tpl', synthetic: true, c: arms },
688
+ ] }];
689
+ }
690
+
691
+ function cloneTree(tokens) {
692
+ return tokens.map(tok => (tok.k === 'group' ? { ...tok, c: cloneTree(tok.c) } : { ...tok }));
693
+ }
694
+
695
+ /**
696
+ * An expansion used as an expression: a block holding a single expression
697
+ * (`{{ Command::new() }}`) is that expression; anything that is not one
698
+ * postfix operand keeps its precedence in parentheses, statements in a block.
699
+ */
700
+ function wrapExpression(input) {
701
+ let tokens = input;
702
+ while (tokens.length === 1 && tokens[0].k === 'group' && tokens[0].d === '{' &&
703
+ tokens[0].c.length > 0 && !tokens[0].c.some(tok => isTok(tok, ';') || isTok(tok, 'let'))) {
704
+ tokens = tokens[0].c;
705
+ }
706
+ if (tokens.length === 1 && tokens[0].k === 'group') return tokens;
707
+ if (isPostfixChain(tokens)) return tokens;
708
+ const hasStatement = tokens.some(tok => isTok(tok, ';'));
709
+ const d = hasStatement ? '{' : '(';
710
+ return [{ k: 'group', v: d, d, c: tokens, src: 'tpl', synthetic: true }];
711
+ }
712
+
713
+ // Hygiene: locals a transcriber introduces (`let x`, closure parameters,
714
+ // `for x in`) live in that expansion step's syntax context and never bind
715
+ // the invocation's tokens of the same spelling (or vice versa).
716
+ const RUST_KEYWORDS = new Set([
717
+ 'as', 'async', 'await', 'break', 'const', 'continue', 'crate', 'dyn', 'else', 'enum', 'extern',
718
+ 'false', 'fn', 'for', 'if', 'impl', 'in', 'let', 'loop', 'match', 'mod', 'move', 'mut', 'pub',
719
+ 'ref', 'return', 'self', 'Self', 'static', 'struct', 'super', 'trait', 'true', 'type', 'unsafe',
720
+ 'use', 'where', 'while', '_',
721
+ ]);
722
+
723
+ function renameHygienicLocals(tokens) {
724
+ const binders = new Set();
725
+ const collect = list => {
726
+ for (let i = 0; i < list.length; i++) {
727
+ const tok = list[i];
728
+ if (tok.k === 'group') {
729
+ collect(tok.c);
730
+ if (isTok(list[i - 1], 'let') || (isTok(list[i - 1], 'mut') && isTok(list[i - 2], 'let'))) {
731
+ for (const inner of tok.c) {
732
+ if (inner.k === 'ident' && inner.src === 'tpl' && !RUST_KEYWORDS.has(inner.v) &&
733
+ !/^[A-Z]/.test(inner.v)) {
734
+ binders.add(`${inner.ctx}\0${inner.v}`);
735
+ }
736
+ }
737
+ }
738
+ continue;
739
+ }
740
+ if (tok.src !== 'tpl' || tok.k !== 'ident' || RUST_KEYWORDS.has(tok.v)) continue;
741
+ const prev = list[i - 1];
742
+ const prev2 = list[i - 2];
743
+ const next = list[i + 1];
744
+ // `let Some(x)` / `let Point { x, .. }` / `let a::B`: a path,
745
+ // not a binding (the bindings sit in the pattern's group).
746
+ const pathHead = (next?.k === 'group' && next.d !== '[') || isTok(next, '::');
747
+ if (pathHead && (isTok(prev, 'let') || (isTok(prev, 'mut') && isTok(prev2, 'let')))) {
748
+ for (const inner of next.c || []) {
749
+ if (inner.k === 'ident' && inner.src === 'tpl' && !RUST_KEYWORDS.has(inner.v) &&
750
+ !/^[A-Z]/.test(inner.v)) binders.add(`${inner.ctx}\0${inner.v}`);
751
+ }
752
+ continue;
753
+ }
754
+ if (isTok(prev, 'let') || (isTok(prev, 'mut') && isTok(prev2, 'let')) ||
755
+ (isTok(prev, 'for') && isTok(next, 'in'))) {
756
+ binders.add(`${tok.ctx}\0${tok.v}`);
757
+ }
758
+ }
759
+ // closure parameters: `|a, mut b: T|`
760
+ for (let i = 0; i < list.length; i++) {
761
+ if (!isTok(list[i], '|') || list[i].src !== 'tpl') continue;
762
+ const prev = list[i - 1];
763
+ if (prev && prev.k !== 'group' && !['(', ',', '=', 'move', '=>', ';'].includes(prev.v) &&
764
+ prev.k !== 'punct') continue;
765
+ let j = i + 1;
766
+ let expectName = true;
767
+ while (j < list.length && !isTok(list[j], '|')) {
768
+ const tok = list[j];
769
+ if (isTok(tok, ',')) expectName = true;
770
+ else if (expectName && tok.k === 'ident' && !RUST_KEYWORDS.has(tok.v)) {
771
+ if (tok.src === 'tpl') binders.add(`${tok.ctx}\0${tok.v}`);
772
+ expectName = false;
773
+ } else if (!isTok(tok, 'mut') && !isTok(tok, 'ref') && !isTok(tok, '&')) {
774
+ expectName = false;
775
+ }
776
+ j++;
777
+ }
778
+ i = j;
779
+ }
780
+ };
781
+ collect(tokens);
782
+ if (binders.size === 0) return tokens;
783
+ const rename = list => {
784
+ for (let i = 0; i < list.length; i++) {
785
+ const tok = list[i];
786
+ if (tok.k === 'group') {
787
+ rename(tok.c);
788
+ continue;
789
+ }
790
+ if (tok.k !== 'ident' || tok.src !== 'tpl' || RUST_KEYWORDS.has(tok.v) ||
791
+ !binders.has(`${tok.ctx}\0${tok.v}`)) continue;
792
+ if (isTok(list[i - 1], '.') || isTok(list[i - 1], '::') ||
793
+ isTok(list[i + 1], '::') || isTok(list[i + 1], '!')) continue;
794
+ tok.v = `${tok.v}__ucn_h${tok.ctx}`;
795
+ }
796
+ };
797
+ rename(tokens);
798
+ return tokens;
799
+ }
800
+
801
+ // ── Layout ────────────────────────────────────────────────────────────────
802
+
803
+ const CLOSE = { '(': ')', '[': ']', '{': '}' };
804
+
805
+ function flatten(tokens, out = []) {
806
+ for (const tok of tokens) {
807
+ if (tok.k === 'group') {
808
+ out.push({ v: tok.d, src: tok.src, line: tok.line, col: tok.col, s: tok.s, e: tok.s !== undefined ? tok.s + 1 : undefined });
809
+ flatten(tok.c, out);
810
+ const close = tok.closeOrigin;
811
+ out.push(close && tok.src === 'arg'
812
+ ? { v: CLOSE[tok.d], src: 'arg', line: close.line, col: close.col, s: close.s, e: close.e, closer: true }
813
+ : { v: CLOSE[tok.d], src: 'tpl', closer: true });
814
+ } else {
815
+ out.push(tok);
816
+ }
817
+ }
818
+ return out;
819
+ }
820
+
821
+ const MULTI_PUNCT = ['::', '->', '=>', '==', '!=', '<=', '>=', '&&', '||', '+=', '-=', '*=', '/=',
822
+ '%=', '^=', '&=', '|=', '<<', '>>', '..', '//', '/*', '*/'];
823
+ // The two-character joints that start a multi-character punctuation token
824
+ // (a one-character joint, from an empty leaf, starts any that begins with it).
825
+ const MULTI_PUNCT_JOINTS = new Set(MULTI_PUNCT.flatMap(p => [p.slice(0, 2), p[0]]));
826
+ const SPACE_AFTER = new Set([',', ';', ':', '=>', '->', '=', '+', '|', '&&', '||', '==', '!=']);
827
+ const SPACE_BEFORE = new Set(['=>', '->', '=', '+', '{', '==', '!=', '&&', '||']);
828
+ // ASCII letters, digits, '_', quotes (and '#' at a word's end): a leaf that
829
+ // ends / starts with one lexes into its neighbor without a space.
830
+ function isWordCode(code) {
831
+ return (code >= 97 && code <= 122) || (code >= 65 && code <= 90) || (code >= 48 && code <= 57) ||
832
+ code === 95 || code === 39 || code === 34;
833
+ }
834
+ const isWordEnd = leaf => {
835
+ const v = typeof leaf.v === 'string' ? leaf.v : String(leaf.v);
836
+ if (v.length === 0) return false;
837
+ const code = v.charCodeAt(v.length - 1);
838
+ return isWordCode(code) || code === 35;
839
+ };
840
+ const isWordStart = leaf => {
841
+ const v = typeof leaf.v === 'string' ? leaf.v : String(leaf.v);
842
+ return v.length > 0 && isWordCode(v.charCodeAt(0));
843
+ };
844
+
845
+ /**
846
+ * Whether two adjacent leaves need a space: when they would lex as one
847
+ * token, plus conventional spacing so text-derived facts (parameter lists,
848
+ * generics, paths) read like source.
849
+ */
850
+ function needsSpace(prev, leaf) {
851
+ if (!prev) return false;
852
+ const wordEnd = isWordEnd(prev);
853
+ const wordStart = isWordStart(leaf);
854
+ if (wordEnd && wordStart) return true;
855
+ const a = prev.v;
856
+ const b = leaf.v;
857
+ if (b === '.') {
858
+ const first = String(a).charCodeAt(0);
859
+ if (first >= 48 && first <= 57) return true;
860
+ }
861
+ if (!wordEnd && !wordStart) {
862
+ const joint = a.slice(-1) + b[0];
863
+ if (MULTI_PUNCT_JOINTS.has(joint) && !(a === '::' || b === '::')) return true;
864
+ }
865
+ if (a === ':' && b === ':') return true;
866
+ if (SPACE_AFTER.has(a) && a !== '::') return !(a === '|' && prev.closer);
867
+ if ((a === '>' || a === '}') && wordStart) return true;
868
+ if (SPACE_BEFORE.has(b)) return true;
869
+ return false;
870
+ }
871
+
872
+ /**
873
+ * Text of an expansion laid out on the invocation's lines [startLine,
874
+ * endLine]: argument tokens move to their own line when it is ahead, leading
875
+ * template tokens join the next argument token's line, nothing ever passes
876
+ * endLine. Returns { text, marks: [{s, e, leaf}] } (offsets relative).
877
+ */
878
+ function layout(tokens, startLine, startCol, endLine) {
879
+ const leaves = flatten(tokens);
880
+ const nextArgLine = new Array(leaves.length + 1).fill(0);
881
+ for (let i = leaves.length - 1; i >= 0; i--) {
882
+ nextArgLine[i] = leaves[i].src === 'arg' ? leaves[i].line : nextArgLine[i + 1];
883
+ }
884
+ let line = startLine;
885
+ let col = startCol;
886
+ let text = '';
887
+ const marks = [];
888
+ let first = true;
889
+ for (let i = 0; i < leaves.length; i++) {
890
+ const leaf = leaves[i];
891
+ let target = 0;
892
+ if (leaf.src === 'arg' && leaf.line > line) target = leaf.line;
893
+ else if (leaf.src !== 'arg' && !leaf.closer && nextArgLine[i + 1] > line) target = nextArgLine[i + 1];
894
+ if (target > line && target <= endLine) {
895
+ text += '\n'.repeat(target - line);
896
+ line = target;
897
+ col = 0;
898
+ if (leaf.src === 'arg' && leaf.col > col) {
899
+ text += ' '.repeat(leaf.col);
900
+ col = leaf.col;
901
+ }
902
+ } else if (!first && needsSpace(leaves[i - 1], leaf)) {
903
+ text += ' ';
904
+ col++;
905
+ }
906
+ first = false;
907
+ let value = leaf.v;
908
+ if (value.includes('\n')) {
909
+ const breaks = value.split('\n').length - 1;
910
+ if (leaf.src === 'arg' && line + breaks <= endLine) {
911
+ line += breaks;
912
+ col = value.length - value.lastIndexOf('\n') - 1;
913
+ } else {
914
+ value = /^b?r?#*"|^b?"/.test(value) ? '""' : ' ';
915
+ col += value.length;
916
+ }
917
+ } else {
918
+ col += value.length;
919
+ }
920
+ marks.push({ s: text.length, e: text.length + value.length, leaf, line });
921
+ text += value;
922
+ }
923
+ if (line < endLine) text += '\n'.repeat(endLine - line);
924
+ return { text, marks };
925
+ }
926
+
927
+ // ── Per-file expansion ────────────────────────────────────────────────────
928
+
929
+ function collectErrorRanges(node, out) {
930
+ if (node.type === 'ERROR' || node.isMissing) {
931
+ out.push([node.startIndex, node.endIndex]);
932
+ return out;
933
+ }
934
+ if (!node.hasError) return out;
935
+ for (let i = 0; i < node.childCount; i++) collectErrorRanges(node.child(i), out);
936
+ return out;
937
+ }
938
+
939
+ /**
940
+ * The expanded file as analyzed: only the kept ranges (units holding an
941
+ * invocation and their ancestors' headers) are text; everything else is
942
+ * blanked with its line breaks kept, so every line keeps its number and the
943
+ * parser only walks what it must re-derive. Expansions replace their
944
+ * invocation's span.
945
+ */
946
+ function buildSynthetic(content, regions, keep) {
947
+ let text = '';
948
+ let cursor = 0;
949
+ const spans = []; // synthetic [start, end) of text that is not blank
950
+ const blank = (from, to) => content.slice(from, to).replace(/[^\r\n]/g, ' ');
951
+ const addSpan = (start, end) => {
952
+ if (end <= start) return;
953
+ const last = spans[spans.length - 1];
954
+ if (last && last[1] >= start) last[1] = Math.max(last[1], end);
955
+ else spans.push([start, end]);
956
+ };
957
+ const copy = (from, to) => {
958
+ // Kept ranges are sorted and disjoint.
959
+ let at = from;
960
+ for (const [start, end] of keep) {
961
+ if (end <= at) continue;
962
+ if (start >= to) break;
963
+ if (start > at) text += blank(at, start);
964
+ const stop = Math.min(end, to);
965
+ const synthStart = text.length;
966
+ text += content.slice(Math.max(start, at), stop);
967
+ addSpan(synthStart, text.length);
968
+ at = stop;
969
+ }
970
+ if (at < to) text += blank(at, to);
971
+ };
972
+ for (const region of regions) {
973
+ copy(cursor, region.start);
974
+ region.synthStart = text.length;
975
+ text += region.layout.text;
976
+ region.synthEnd = text.length;
977
+ // A stand-in for an expansion another group reads keeps its length.
978
+ if (!region.blank) addSpan(region.synthStart, region.synthEnd);
979
+ cursor = region.end;
980
+ }
981
+ copy(cursor, content.length);
982
+ return { text, spans };
983
+ }
984
+
985
+ /**
986
+ * Parse only the non-blank spans of the synthetic file (tree-sitter
987
+ * included ranges): positions stay absolute, blank text costs nothing. The
988
+ * tree lands in safeParse's cache, so the adapter's analysis reuses it.
989
+ */
990
+ function parseSynthetic(parser, text, spans) {
991
+ const starts = [0];
992
+ for (let i = text.indexOf('\n'); i >= 0; i = text.indexOf('\n', i + 1)) starts.push(i + 1);
993
+ const position = offset => {
994
+ let lo = 0;
995
+ let hi = starts.length - 1;
996
+ while (lo < hi) {
997
+ const mid = (lo + hi + 1) >> 1;
998
+ if (starts[mid] <= offset) lo = mid; else hi = mid - 1;
999
+ }
1000
+ return { row: lo, column: offset - starts[lo] };
1001
+ };
1002
+ const includedRanges = spans.map(([start, end]) => ({
1003
+ startIndex: start, endIndex: end, startPosition: position(start), endPosition: position(end),
1004
+ }));
1005
+ if (includedRanges.length === 0) return safeParse(parser, text, undefined, PARSE_OPTIONS);
1006
+ return safeParse(parser, text, undefined, { ...PARSE_OPTIONS, includedRanges });
1007
+ }
1008
+
1009
+ const UNIT_TYPES = new Set(['function_item', 'const_item', 'static_item']);
1010
+ const HEADER_TYPES = new Set(['function_item', 'impl_item', 'trait_item', 'mod_item', 'foreign_mod_item']);
1011
+
1012
+ /** The node plus its preceding attributes and comments. */
1013
+ function withLeadingTrivia(node) {
1014
+ let start = node.startIndex;
1015
+ let startRow = node.startPosition.row;
1016
+ for (let sibling = node.previousSibling; sibling; sibling = sibling.previousSibling) {
1017
+ if (sibling.type !== 'attribute_item' && sibling.type !== 'line_comment' &&
1018
+ sibling.type !== 'block_comment') break;
1019
+ start = sibling.startIndex;
1020
+ startRow = sibling.startPosition.row;
1021
+ }
1022
+ return { start, startLine: startRow + 1 };
1023
+ }
1024
+
1025
+ /**
1026
+ * Units re-derived from the expanded file: the innermost function (or
1027
+ * const/static item) holding each invocation, else the item-position
1028
+ * invocation itself, with leading attributes; plus, kept as text, the
1029
+ * headers and associated types of every enclosing impl/trait/mod/fn so the
1030
+ * unit parses in its real context (Self type, trait, enclosing fn).
1031
+ */
1032
+ function analysisUnits(regions, mode = 'calls') {
1033
+ const units = [];
1034
+ const keep = [];
1035
+ const headers = new Set();
1036
+ const keepHeader = parent => {
1037
+ if (headers.has(parent.startIndex)) return;
1038
+ headers.add(parent.startIndex);
1039
+ const body = parent.childForFieldName('body');
1040
+ if (!body) return;
1041
+ keep.push([parent.startIndex, body.startIndex + 1], [body.endIndex - 1, body.endIndex]);
1042
+ // Associated types of an enclosing impl/trait.
1043
+ if (parent.type === 'impl_item' || parent.type === 'trait_item') {
1044
+ for (const member of body.namedChildren) {
1045
+ if (member.type === 'type_item' || member.type === 'associated_type') {
1046
+ keep.push([member.startIndex, member.endIndex]);
1047
+ }
1048
+ }
1049
+ }
1050
+ const { start: headStart } = withLeadingTrivia(parent);
1051
+ if (headStart < parent.startIndex) keep.push([headStart, parent.startIndex]);
1052
+ };
1053
+ for (const region of regions) {
1054
+ let unitNode = null;
1055
+ for (let parent = region.node.parent; parent; parent = parent.parent) {
1056
+ if (UNIT_TYPES.has(parent.type)) { unitNode = parent; break; }
1057
+ }
1058
+ let start;
1059
+ let startLine;
1060
+ let end;
1061
+ let endLine;
1062
+ if (unitNode) {
1063
+ ({ start, startLine } = withLeadingTrivia(unitNode));
1064
+ end = unitNode.endIndex;
1065
+ endLine = unitNode.endPosition.row + 1;
1066
+ } else {
1067
+ ({ start, startLine } = withLeadingTrivia(region.node));
1068
+ const next = region.node.nextSibling;
1069
+ const last = next && next.type === ';' ? next : region.node;
1070
+ end = last.endIndex;
1071
+ endLine = last.endPosition.row + 1;
1072
+ }
1073
+ units.push({ start, end, startLine, endLine });
1074
+ if (mode === 'symbols' && unitNode) {
1075
+ // Declarations need only the invocation in its enclosing
1076
+ // function's frame, not the rest of the body.
1077
+ if (unitNode.type === 'function_item') keepHeader(unitNode);
1078
+ else keep.push([start, end]);
1079
+ const next = region.node.nextSibling;
1080
+ keep.push([region.start, next && next.type === ';' ? next.endIndex : region.end]);
1081
+ } else {
1082
+ keep.push([start, end]);
1083
+ }
1084
+ for (let parent = (unitNode || region.node).parent; parent; parent = parent.parent) {
1085
+ if (HEADER_TYPES.has(parent.type)) keepHeader(parent);
1086
+ }
1087
+ }
1088
+ const merge = ranges => {
1089
+ ranges.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
1090
+ const out = [];
1091
+ for (const range of ranges) {
1092
+ const last = out[out.length - 1];
1093
+ if (last && range[0] <= last[1]) last[1] = Math.max(last[1], range[1]);
1094
+ else out.push([...range]);
1095
+ }
1096
+ return out;
1097
+ };
1098
+ units.sort((a, b) => a.start - b.start || b.end - a.end);
1099
+ const merged = [];
1100
+ for (const unit of units) {
1101
+ const last = merged[merged.length - 1];
1102
+ if (last && unit.start <= last.end) {
1103
+ if (unit.end > last.end) { last.end = unit.end; last.endLine = unit.endLine; }
1104
+ } else merged.push({ ...unit });
1105
+ }
1106
+ return { units: merged, keep: merge(keep) };
1107
+ }
1108
+
1109
+ function regionAt(regions, synth) {
1110
+ let lo = 0;
1111
+ let hi = regions.length - 1;
1112
+ while (lo <= hi) {
1113
+ const mid = (lo + hi) >> 1;
1114
+ const region = regions[mid];
1115
+ if (synth < region.synthStart) hi = mid - 1;
1116
+ else if (synth >= region.synthEnd) lo = mid + 1;
1117
+ else return region;
1118
+ }
1119
+ return null;
1120
+ }
1121
+
1122
+ /**
1123
+ * Mark of the leaf at relative offset `rel`: the leaf containing it, else
1124
+ * (whitespace) the next leaf for a start offset, the previous for an end.
1125
+ */
1126
+ function markAt(marks, rel, forEnd) {
1127
+ const probe = forEnd ? rel - 1 : rel;
1128
+ let lo = 0;
1129
+ let hi = marks.length - 1;
1130
+ while (lo <= hi) {
1131
+ const mid = (lo + hi) >> 1;
1132
+ const mark = marks[mid];
1133
+ if (probe < mark.s) hi = mid - 1;
1134
+ else if (probe >= mark.e) lo = mid + 1;
1135
+ else return mark;
1136
+ }
1137
+ // lo = first mark starting after probe, hi = last mark ending before it
1138
+ return forEnd ? (marks[hi] || marks[lo] || null) : (marks[lo] || marks[hi] || null);
1139
+ }
1140
+
1141
+ /** Maps synthetic offsets back to the source file. */
1142
+ function offsetMapper(regions) {
1143
+ // Length change accumulated after each region.
1144
+ let delta = 0;
1145
+ for (const region of regions) {
1146
+ delta += (region.synthEnd - region.synthStart) - (region.end - region.start);
1147
+ region.deltaAfter = delta;
1148
+ }
1149
+ const map = (synth, forEnd) => {
1150
+ const region = regionAt(regions, forEnd ? synth - 1 : synth);
1151
+ if (!region) {
1152
+ // Copied text: shift by every region before it.
1153
+ let lo = 0;
1154
+ let hi = regions.length - 1;
1155
+ let before = null;
1156
+ while (lo <= hi) {
1157
+ const mid = (lo + hi) >> 1;
1158
+ if (regions[mid].synthEnd <= synth) { before = regions[mid]; lo = mid + 1; } else hi = mid - 1;
1159
+ }
1160
+ return synth - (before ? before.deltaAfter : 0);
1161
+ }
1162
+ const rel = synth - region.synthStart;
1163
+ const mark = markAt(region.layout.marks, rel, forEnd);
1164
+ if (mark && mark.leaf.src === 'arg' && Number.isInteger(mark.leaf.s)) {
1165
+ return mark.leaf.s + Math.min(Math.max(rel - mark.s, 0), mark.e - mark.s);
1166
+ }
1167
+ return fraction(region, synth);
1168
+ };
1169
+ // Template text has no source offset: a stable fraction inside the
1170
+ // invocation's first byte keeps order and receiver links.
1171
+ const fraction = (region, synth) =>
1172
+ region.start + (synth - region.synthStart + 1) / (region.synthEnd - region.synthStart + 2);
1173
+ const templateOffset = synth => {
1174
+ const region = regionAt(regions, synth) || regionAt(regions, synth - 1);
1175
+ return region ? fraction(region, Math.min(synth, region.synthEnd)) : map(synth, false);
1176
+ };
1177
+ return { map, templateOffset, regionAt: synth => regionAt(regions, synth) };
1178
+ }
1179
+
1180
+ const OFFSET_FIELD = /(?:Start|End|^start|^end)$/;
1181
+
1182
+ /**
1183
+ * Remap every `<x>Start`/`<x>End` offset field. A span keeps source offsets
1184
+ * only when both ends were written in the source (outside expansions, or
1185
+ * both inside argument tokens); otherwise both ends take the invocation's
1186
+ * fractional template offsets, so a span never inverts.
1187
+ */
1188
+ function remapOffsets(record, mapper, depth = 0) {
1189
+ const pairs = new Map();
1190
+ for (const key of Object.keys(record)) {
1191
+ const value = record[key];
1192
+ if (value && typeof value === 'object' && depth < 4) {
1193
+ // Nested evidence (receiverTypeEvidence, origins) carries offsets too.
1194
+ if (Array.isArray(value)) {
1195
+ record[key] = value.map(item => {
1196
+ if (!item || typeof item !== 'object') return item;
1197
+ const copy = { ...item };
1198
+ remapOffsets(copy, mapper, depth + 1);
1199
+ return copy;
1200
+ });
1201
+ } else {
1202
+ const copy = { ...value };
1203
+ remapOffsets(copy, mapper, depth + 1);
1204
+ record[key] = copy;
1205
+ }
1206
+ continue;
1207
+ }
1208
+ if (typeof value !== 'number' || !OFFSET_FIELD.test(key)) continue;
1209
+ const stem = key.replace(OFFSET_FIELD, '');
1210
+ if (!pairs.has(stem)) pairs.set(stem, {});
1211
+ pairs.get(stem)[/end$/i.test(key) ? 'end' : 'start'] = key;
1212
+ }
1213
+ for (const { start, end } of pairs.values()) {
1214
+ const mappedStart = start ? mapper.map(record[start], false) : null;
1215
+ const mappedEnd = end ? mapper.map(record[end], true) : null;
1216
+ const mixed = start && end &&
1217
+ (Number.isInteger(mappedStart) !== Number.isInteger(mappedEnd) || mappedEnd < mappedStart);
1218
+ if (mixed) {
1219
+ record[start] = mapper.templateOffset(record[start]);
1220
+ record[end] = mapper.templateOffset(record[end]);
1221
+ continue;
1222
+ }
1223
+ if (start) record[start] = mappedStart;
1224
+ if (end) record[end] = mappedEnd;
1225
+ }
1226
+ }
1227
+
1228
+ const DECLARATION_TYPES = [
1229
+ 'function_item', 'function_signature_item', 'impl_item', 'struct_item', 'enum_item', 'union_item',
1230
+ 'trait_item', 'mod_item', 'type_item', 'const_item', 'static_item', 'macro_definition',
1231
+ 'field_declaration', 'enum_variant', 'associated_type',
1232
+ ];
1233
+
1234
+ const DECLARATION_TYPE_SET = new Set(DECLARATION_TYPES);
1235
+ let declarationWalkTypes = null;
1236
+ function declarationWalkTypesOf() {
1237
+ return declarationWalkTypes || (declarationWalkTypes =
1238
+ [...new Set([...DECLARATION_TYPES, ...require('../languages/rust').DECLARATION_NODE_TYPES])]);
1239
+ }
1240
+
1241
+ /**
1242
+ * (startLine\0name) keys of declarations whose name was produced inside a
1243
+ * region. `within`: the roots spanning the regions and their walked
1244
+ * descendants, when every such declaration lies inside one (else the whole
1245
+ * tree is searched).
1246
+ */
1247
+ function generatedDeclarations(root, regions, within = null) {
1248
+ const out = new Map();
1249
+ if (regions.length === 0) return out;
1250
+ const nodes = within
1251
+ ? within.roots.flatMap((spanning, i) => [
1252
+ ...(DECLARATION_TYPE_SET.has(spanning.type) ? [spanning] : []),
1253
+ ...within.walked[i].filter(node => DECLARATION_TYPE_SET.has(node.type)),
1254
+ ])
1255
+ : root.descendantsOfType(DECLARATION_TYPES);
1256
+ for (const node of nodes) {
1257
+ const nameNode = node.type === 'impl_item' ? node.childForFieldName('type')
1258
+ : node.childForFieldName('name');
1259
+ if (!nameNode) continue;
1260
+ const region = regionAt(regions, nameNode.startIndex);
1261
+ if (!region || nameNode.endIndex > region.synthEnd) continue;
1262
+ const mark = markAt(region.layout.marks, nameNode.startIndex - region.synthStart, false);
1263
+ const origin = mark?.leaf.src === 'arg' ? 'argument' : 'template';
1264
+ const text = node.type === 'impl_item' ? typeNameOf(nameNode) : nameNode.text;
1265
+ out.set(`${node.startPosition.row + 1}\0${text}`, {
1266
+ region, origin,
1267
+ ...(origin === 'argument' && { nameLine: mark.leaf.line }),
1268
+ });
1269
+ }
1270
+ return out;
1271
+ }
1272
+
1273
+ function typeNameOf(node) {
1274
+ if (node.type === 'generic_type') return typeNameOf(node.childForFieldName('type'));
1275
+ if (node.type === 'scoped_type_identifier') return node.childForFieldName('name')?.text || node.text;
1276
+ if (node.type === 'reference_type') return typeNameOf(node.childForFieldName('type'));
1277
+ return node.text;
1278
+ }
1279
+
1280
+ function macroFact(region, origin) {
1281
+ return {
1282
+ macro: region.macro,
1283
+ definition: region.definition,
1284
+ origin,
1285
+ line: region.startLine,
1286
+ ...(region.ambiguous && { ambiguous: true }),
1287
+ };
1288
+ }
1289
+
1290
+ /**
1291
+ * The invocations of a file the expansion pass reads, in document order:
1292
+ * project macros in scope that are worth expanding. Invocations without a
1293
+ * context or token tree are recorded blind in `rec`.
1294
+ */
1295
+ function plannedInvocations(state, filePath, content, tree, rec, invocations = null) {
1296
+ const plan = [];
1297
+ for (const node of invocations || tree.rootNode.descendantsOfType('macro_invocation')) {
1298
+ const id = invocationName(node);
1299
+ if (!id || !state.inventory.has(id.name)) continue;
1300
+ let inDefinition = false;
1301
+ for (let parent = node.parent; parent; parent = parent.parent) {
1302
+ if (parent.type === 'macro_definition') { inDefinition = true; break; }
1303
+ }
1304
+ if (inDefinition) continue;
1305
+ const line = node.startPosition.row + 1;
1306
+ const resolved = resolveMacro(state, filePath, node.startIndex, line, id.name, id.qualifier, null);
1307
+ if (!resolved) continue;
1308
+ const def = resolved.defs[0];
1309
+ const info = compiledDef(state, def);
1310
+ const context = invocationContext(node);
1311
+ if (!worthExpanding(state, resolved.defs, content.slice(node.startIndex, node.endIndex),
1312
+ context === 'expr')) continue;
1313
+ const tokenTree = node.namedChildren.find(child => child.type === 'token_tree');
1314
+ if (info.compiled?.passThroughItems && context === 'items' && itemRecoveryPosition(node)) continue;
1315
+ if (!context) { rec.blind.push({ line, macro: id.name, reason: 'context' }); continue; }
1316
+ if (!tokenTree) { rec.blind.push({ line, macro: id.name, reason: 'token-error' }); continue; }
1317
+ plan.push({ node, id, resolved, def, context, tokenTree, line });
1318
+ }
1319
+ return plan;
1320
+ }
1321
+
1322
+ /** Expand one planned invocation: its region, or null (recorded blind). */
1323
+ function expandPlanned(state, filePath, content, item, rec) {
1324
+ const { node, id, resolved, def, context, tokenTree, line } = item;
1325
+ const site = {
1326
+ file: filePath, offset: node.startIndex, line, budget: MAX_OUTPUT_TOKENS,
1327
+ used: rec.used, nestedBlind: [], ambiguous: resolved.ambiguous,
1328
+ };
1329
+ let tokens;
1330
+ try {
1331
+ lineOffset(state, filePath, 1);
1332
+ const toks = argTokensOf(tokenTree, content, state.lineStarts.get(filePath));
1333
+ tokens = resolved.alternatives
1334
+ ? expandAlternatives(state, site, resolved, toks, 0, context)
1335
+ : expandTokens(state, site, def, toks, 0);
1336
+ } catch (error) {
1337
+ if (!(error instanceof MacroBlind)) throw error;
1338
+ rec.blind.push({ line, macro: id.name, reason: error.reason });
1339
+ return null;
1340
+ }
1341
+ for (const nested of site.nestedBlind) rec.blind.push({ line, macro: nested.macro, reason: `nested:${nested.reason}` });
1342
+ renameHygienicLocals(tokens);
1343
+ let body = tokens;
1344
+ if (context === 'expr') body = wrapExpression(tokens);
1345
+ const startLine = line;
1346
+ const endLine = node.endPosition.row + 1;
1347
+ const ancestors = new Set();
1348
+ for (let parent = node.parent; parent; parent = parent.parent) {
1349
+ const nameNode = parent.type === 'impl_item' ? parent.childForFieldName('type')
1350
+ : DECLARATION_TYPES.includes(parent.type) ? parent.childForFieldName('name') : null;
1351
+ if (nameNode) {
1352
+ ancestors.add(`${parent.startPosition.row + 1}\0${parent.type === 'impl_item' ? typeNameOf(nameNode) : nameNode.text}`);
1353
+ }
1354
+ }
1355
+ return {
1356
+ start: node.startIndex, end: node.endIndex, startLine, endLine,
1357
+ macro: id.name, definition: defKey(def), ambiguous: site.ambiguous, ancestors,
1358
+ node, layout: layout(body, startLine, node.startPosition.column, endLine),
1359
+ };
1360
+ }
1361
+
1362
+ /**
1363
+ * Expand the project macro invocations of one file. Returns the expansion
1364
+ * record, or null when the file invokes no project macro in scope.
1365
+ */
1366
+ function expandFile(state, filePath, mode = 'symbols') {
1367
+ // Hygiene contexts are numbered per file, so an expansion's renamed
1368
+ // locals do not depend on what else this state expanded before.
1369
+ state.hygieneSeq = 0;
1370
+ const index = state.index;
1371
+ const fileEntry = index.files.get(filePath);
1372
+ const content = readContent(state, filePath);
1373
+ if (!fileEntry || content === null) return null;
1374
+ if (!fileMayExpand(state, filePath)) return { sites: 0, blind: [], used: new Map(), expanded: false };
1375
+ const tree = safeParse(state.parser, content, undefined, PARSE_OPTIONS);
1376
+ const rec = { sites: 0, blind: [], used: new Map(), expanded: false };
1377
+ const regions = [];
1378
+ // Each expansion's hygiene start and expanded length (fix #405): the
1379
+ // calls pass expands one group of units with the same numbering and
1380
+ // the same offsets as a pass over the whole file.
1381
+ const expansions = mode === 'symbols' ? [] : null;
1382
+ // The symbols pass also needs the file's literals and comments: one
1383
+ // walk lists both (fix #405b).
1384
+ const clean = !tree.rootNode.hasError;
1385
+ const walked = mode === 'symbols' && clean
1386
+ ? tree.rootNode.descendantsOfType(['macro_invocation', ...NON_CODE_TOKENS]) : null;
1387
+ const invocations = walked ? walked.filter(node => node.type === 'macro_invocation') : null;
1388
+ const nonCode = walked ? walked.filter(node => node.type !== 'macro_invocation') : [];
1389
+ for (const item of plannedInvocations(state, filePath, content, tree, rec, invocations)) {
1390
+ const hygiene = state.hygieneSeq;
1391
+ const region = expandPlanned(state, filePath, content, item, rec);
1392
+ if (expansions) expansions.push(item.node.startIndex, hygiene, region ? region.layout.text.length : -1);
1393
+ if (region) regions.push(region);
1394
+ }
1395
+ rec.sites = regions.length;
1396
+ if (regions.length === 0) return rec;
1397
+ regions.sort((a, b) => a.start - b.start);
1398
+ // The identifiers each expansion outputs.
1399
+ const outputs = mode === 'symbols' ? regions.map(outputWordsOf) : null;
1400
+ // Every word the expanded text of the calls pass can hold (fix #375).
1401
+ const words = mode === 'symbols' ? expansionWords(content, regions, nonCode, outputs) : null;
1402
+ // What each group of units can bear, read before parsing it (fix #405b).
1403
+ const groups = mode === 'symbols' && !tree.rootNode.hasError
1404
+ ? groupRecords(content, regions, nonCode, outputs) : null;
1405
+
1406
+ // Parse the expanded file; an expansion that does not parse in its
1407
+ // context stays as written (blind), and the rest are parsed again.
1408
+ let synthetic;
1409
+ let synthTree;
1410
+ let units;
1411
+ for (let attempt = 0; attempt <= regions.length && regions.length > 0; attempt++) {
1412
+ const scope = analysisUnits(regions, mode);
1413
+ units = scope.units;
1414
+ const built = buildSynthetic(content, regions, scope.keep);
1415
+ synthetic = built.text;
1416
+ synthTree = parseSynthetic(state.parser, synthetic, built.spans);
1417
+ const errors = collectErrorRanges(synthTree.rootNode, []);
1418
+ const bad = new Set();
1419
+ for (const [start, end] of errors) {
1420
+ for (const region of regions) {
1421
+ if (start < region.synthEnd && end > region.synthStart) bad.add(region);
1422
+ else if (start === end && start >= region.synthStart && start <= region.synthEnd) bad.add(region);
1423
+ }
1424
+ }
1425
+ if (bad.size === 0) break;
1426
+ for (const region of bad) {
1427
+ rec.blind.push({ line: region.startLine, macro: region.macro, reason: 'parse-error' });
1428
+ regions.splice(regions.indexOf(region), 1);
1429
+ }
1430
+ }
1431
+ rec.sites = regions.length;
1432
+ if (regions.length === 0) return rec;
1433
+
1434
+ const rust = require('../languages/rust');
1435
+ const mapper = offsetMapper(regions);
1436
+ const lookups = unitLookups(units);
1437
+ rec.expanded = true;
1438
+
1439
+ if (mode === 'symbols') {
1440
+ // The expanded file holds no unexpanded item bodies worth recovering
1441
+ // inside its units; the extractors share the included-ranges tree.
1442
+ rust.primeDeclarationTrees(synthetic, state.parser, synthTree);
1443
+ // Declarations of the regions only: the smallest nodes spanning each.
1444
+ const spanning = regions.map(region => (region.synthEnd > region.synthStart
1445
+ ? synthTree.rootNode.descendantForIndex(region.synthStart, region.synthEnd - 1) : null))
1446
+ .filter(Boolean).sort((a, b) => a.startIndex - b.startIndex || b.endIndex - a.endIndex);
1447
+ const roots = [];
1448
+ for (const node of spanning) {
1449
+ const last = roots[roots.length - 1];
1450
+ if (last && node.startIndex >= last.startIndex && node.endIndex <= last.endIndex) continue;
1451
+ roots.push(node);
1452
+ }
1453
+ // One walk of each root serves both extractions (fix #405b). In a
1454
+ // file that parses cleanly an expansion stands as whole items,
1455
+ // statements or a wrapped expression, so a declaration named inside
1456
+ // it lies inside the root spanning it; otherwise the whole tree is
1457
+ // searched.
1458
+ const walked = roots.map(root => root.descendantsOfType(declarationWalkTypesOf()));
1459
+ rec.symbols = generatedSymbols(filePath, rust.parseDeclarationsIn(synthetic, roots, {
1460
+ descendants: (root, i) => walked[i],
1461
+ }), {
1462
+ regions, mapper, generated: generatedDeclarations(synthTree.rootNode, regions,
1463
+ tree.rootNode.hasError ? null : { roots, walked }),
1464
+ });
1465
+ rec.calleeNames = callableNames(regions);
1466
+ rec.expansions = expansions;
1467
+ // Groups as the calls pass reads them apart: only when no expansion
1468
+ // was left blind by a parse error (the pass then reads the file in
1469
+ // one pass, whose groups can differ).
1470
+ rec.words = words;
1471
+ if (groups && !rec.blind.some(site => site.reason === 'parse-error')) rec.groups = groups;
1472
+ rec.callsPending = true;
1473
+ return rec;
1474
+ }
1475
+
1476
+ // Calls: records of the units from the expanded file, the rest as parsed,
1477
+ // plus the invocations themselves (a derivation context, fix #405).
1478
+ rec.context = callsContext(state, filePath, {
1479
+ content, tree, regions, units, synthetic, synthTree, mapper, lookups,
1480
+ });
1481
+ return rec;
1482
+ }
1483
+
1484
+ /** Binary-search lookups over sorted, disjoint units. */
1485
+ function unitLookups(units) {
1486
+ const unitAtOffset = offset => {
1487
+ let lo = 0;
1488
+ let hi = units.length - 1;
1489
+ while (lo <= hi) {
1490
+ const mid = (lo + hi) >> 1;
1491
+ if (offset < units[mid].start) hi = mid - 1;
1492
+ else if (offset >= units[mid].end) lo = mid + 1;
1493
+ else return units[mid];
1494
+ }
1495
+ return null;
1496
+ };
1497
+ const unitAtLine = line => byLine(units, line);
1498
+ const inUnit = record => (Number.isFinite(record.callStart) ? !!unitAtOffset(record.callStart)
1499
+ : !!unitAtLine(record.line));
1500
+ return { unitAtOffset, unitAtLine, inUnit };
1501
+ }
1502
+
1503
+ function byLine(list, line) {
1504
+ let lo = 0;
1505
+ let hi = list.length - 1;
1506
+ while (lo <= hi) {
1507
+ const mid = (lo + hi) >> 1;
1508
+ if (line < list[mid].startLine) hi = mid - 1;
1509
+ else if (line > list[mid].endLine) lo = mid + 1;
1510
+ else return list[mid];
1511
+ }
1512
+ return null;
1513
+ }
1514
+
1515
+ // ── Calls pass (fix #405) ─────────────────────────────────────────────────
1516
+ //
1517
+ // The calls of an expanded file are the parsed records outside its units,
1518
+ // the records each unit yields in the expanded file, and the invocations
1519
+ // themselves. Units sharing a line form one group (records without a span
1520
+ // are placed by line). The extractor keeps no scope across functions, so a
1521
+ // group's records do not depend on the other groups: a query that needs
1522
+ // the records bearing one name, or those of a line range, reads only the
1523
+ // groups that can hold them, and the file's calls are composed, as one pass
1524
+ // derives them, when a consumer needs them all. Where the build recorded
1525
+ // every expansion's hygiene start and expanded length, a group expands and
1526
+ // parses alone with the numbering and the offsets of the whole file;
1527
+ // otherwise the file's expansions parse in one pass and each group's
1528
+ // records are read from that tree with the other units skipped.
1529
+
1530
+ /**
1531
+ * Everything the calls pass of a file shares: its regions, units and their
1532
+ * groups, the expanded file and its tree. Per-group records are added as
1533
+ * groups are read.
1534
+ */
1535
+ function callsContext(state, filePath, parts) {
1536
+ const { content, tree, regions, units, synthetic, synthTree, mapper, lookups } = parts;
1537
+ const index = state.index;
1538
+ // Units sharing a line form one group.
1539
+ const groups = groupsOf(units);
1540
+ // Unit spans in the expanded file: a unit starts and ends outside every
1541
+ // region, shifted by the regions before it.
1542
+ const toSynth = offset => {
1543
+ let lo = 0;
1544
+ let hi = regions.length - 1;
1545
+ let before = null;
1546
+ while (lo <= hi) {
1547
+ const mid = (lo + hi) >> 1;
1548
+ if (regions[mid].end <= offset) { before = regions[mid]; lo = mid + 1; } else hi = mid - 1;
1549
+ }
1550
+ return offset + (before ? before.deltaAfter : 0);
1551
+ };
1552
+ for (const unit of units) {
1553
+ unit.synthStart = toSynth(unit.start);
1554
+ unit.synthEnd = toSynth(unit.end);
1555
+ }
1556
+ const cached = index.callsCache.get(filePath);
1557
+ const original = cached?.calls || [];
1558
+ // Records without a source span (struct-literal constructors) are placed
1559
+ // by line: one the unexpanded parse also has on that line is ordinary
1560
+ // code, any other one on an invocation's lines was generated.
1561
+ const originalSpanless = new Map();
1562
+ for (const record of original) {
1563
+ if (Number.isFinite(record.callStart) || !lookups.unitAtLine(record.line)) continue;
1564
+ const key = spanlessKey(record);
1565
+ originalSpanless.set(key, (originalSpanless.get(key) || 0) + 1);
1566
+ }
1567
+ const rust = require('../languages/rust');
1568
+ return {
1569
+ filePath, content, tree, regions, units, groups, synthetic, synthTree, mapper, lookups,
1570
+ original, originalSpanless, seen: new Set(),
1571
+ itemRecovery: rust.rustExpandedItemRecovery(synthetic, state.parser, synthTree),
1572
+ // An expanded file whose parse holds an error is read in one pass.
1573
+ groupwise: !synthTree.rootNode.hasError,
1574
+ invocations: new Map(),
1575
+ parser: state.parser,
1576
+ live: true,
1577
+ };
1578
+ }
1579
+
1580
+ function spanlessKey(record) {
1581
+ return `${record.name}\0${record.line}\0${record.isConstructor ? 1 : 0}`;
1582
+ }
1583
+
1584
+ /** Read the records of the groups `wanted` (indices) not read yet. */
1585
+ function readGroups(ctx, wanted) {
1586
+ let targets = wanted.filter(i => ctx.groups[i].records === null);
1587
+ if (targets.length === 0) return;
1588
+ if (ctx.isolated) {
1589
+ readIsolatedGroups(ctx, targets);
1590
+ ctx.readLookups = null;
1591
+ return;
1592
+ }
1593
+ if (!ctx.groupwise) targets = ctx.groups.map((group, i) => i).filter(i => ctx.groups[i].records === null);
1594
+ const target = new Set(targets);
1595
+ const all = target.size === ctx.groups.filter(group => group.records === null).length &&
1596
+ ctx.groups.every(group => group.records === null);
1597
+ const units = ctx.units;
1598
+ // Nodes inside a unit of another group are not read.
1599
+ const skip = all ? null : node => {
1600
+ const start = node.startIndex;
1601
+ let lo = 0;
1602
+ let hi = units.length - 1;
1603
+ while (lo <= hi) {
1604
+ const mid = (lo + hi) >> 1;
1605
+ const unit = units[mid];
1606
+ if (start < unit.synthStart) hi = mid - 1;
1607
+ else if (start >= unit.synthEnd) lo = mid + 1;
1608
+ else return node.endIndex <= unit.synthEnd && !target.has(unit.group);
1609
+ }
1610
+ return false;
1611
+ };
1612
+ for (const i of targets) ctx.groups[i].records = [];
1613
+ const rust = require('../languages/rust');
1614
+ const synthCalls = rust.findCallsInCode(ctx.synthetic, ctx.parser, {
1615
+ tree: ctx.synthTree, itemRecovery: ctx.itemRecovery, skip,
1616
+ });
1617
+ mergeRecords(ctx, synthCalls, {
1618
+ mapper: ctx.mapper, lookups: ctx.lookups, regions: ctx.regions, synthTree: ctx.synthTree,
1619
+ originalSpanless: ctx.originalSpanless,
1620
+ accept: unit => target.has(unit.group), push: (unit, call) => ctx.groups[unit.group].records.push(call),
1621
+ });
1622
+ }
1623
+
1624
+ /**
1625
+ * The records an expanded file's parse yields for the accepted units, with
1626
+ * source offsets, expansion facts, and repeats dropped (`ctx.seen`).
1627
+ */
1628
+ function mergeRecords(ctx, synthCalls, { mapper, lookups, regions, synthTree, originalSpanless, accept, push }) {
1629
+ const regionOnLine = line => byLine(regions, line);
1630
+ for (const record of synthCalls) {
1631
+ const synthStart = record.callStart;
1632
+ let unit;
1633
+ if (Number.isFinite(synthStart)) {
1634
+ const mapped = mapper.map(synthStart, false);
1635
+ unit = lookups.unitAtOffset(Math.floor(mapped));
1636
+ } else unit = lookups.unitAtLine(record.line);
1637
+ if (!unit || !accept(unit)) continue;
1638
+ const call = { ...record };
1639
+ let region = Number.isFinite(synthStart) ? mapper.regionAt(synthStart) : null;
1640
+ if (!Number.isFinite(synthStart) && regionOnLine(record.line)) {
1641
+ const key = spanlessKey(record);
1642
+ const left = originalSpanless.get(key) || 0;
1643
+ if (left > 0) originalSpanless.set(key, left - 1);
1644
+ else region = regionOnLine(record.line);
1645
+ }
1646
+ remapOffsets(call, mapper);
1647
+ if (region) {
1648
+ // Origin of the NAME token: `v.iter().collect()` written with a
1649
+ // receiver from the arguments still calls a template `collect`.
1650
+ const nameMark = Number.isFinite(synthStart) ? nameMarkOf(synthTree.rootNode, region, record)
1651
+ : region.layout.marks.find(mark => mark.leaf.v === record.name && mark.leaf.src === 'arg' &&
1652
+ mark.line === record.line) || null;
1653
+ const origin = nameMark?.leaf.src === 'arg' ? 'argument' : 'template';
1654
+ // An argument token the layout could not keep on its own line
1655
+ // (repeated or reordered by the transcriber) reports its own.
1656
+ if (origin === 'argument' && nameMark.line !== nameMark.leaf.line) call.line = nameMark.leaf.line;
1657
+ call.macroExpansion = macroFact(region, origin);
1658
+ // Repeated argument tokens are one call; template calls at one
1659
+ // line are told apart only by what the engine can see of them
1660
+ // (a transcriber repetition emits the same call per iteration).
1661
+ const key = origin === 'argument'
1662
+ ? `a\0${call.name}\0${call.line}\0${call.callStart}\0${call.callEnd}\0${call.receiver || ''}`
1663
+ : `t\0${call.name}\0${call.line}\0${call.isMethod ? 1 : 0}\0${call.receiver || ''}\0` +
1664
+ `${call.receiverType || ''}\0${call.receiverRootType || ''}\0${call.receiverField || ''}\0` +
1665
+ `${call.enclosingFunction?.name || ''}\0${call.argCount ?? ''}`;
1666
+ if (ctx.seen.has(key)) continue;
1667
+ ctx.seen.add(key);
1668
+ }
1669
+ push(unit, call);
1670
+ }
1671
+ }
1672
+
1673
+ /**
1674
+ * The invocations stay calls of their macros; the value one produces is the
1675
+ * expansion's, so its own value-flow facts are dropped. Built once per
1676
+ * context (the same records in every view and in the composed calls).
1677
+ */
1678
+ function invocationRecords(ctx) {
1679
+ // Expanded invocations: every region (one pass), the regions of the
1680
+ // groups read so far (groups read apart).
1681
+ const starts = new Set();
1682
+ if (ctx.isolated) {
1683
+ for (const group of ctx.groups) for (const start of group.starts || []) starts.add(start);
1684
+ } else {
1685
+ for (const region of ctx.regions) starts.add(region.start);
1686
+ }
1687
+ const out = [];
1688
+ for (const record of ctx.original) {
1689
+ if (!record.isMacro || !starts.has(record.callStart)) continue;
1690
+ let invocation = ctx.invocations.get(record);
1691
+ if (!invocation) {
1692
+ invocation = {
1693
+ name: record.name, line: record.line, callStart: record.callStart, callEnd: record.callEnd,
1694
+ isMethod: false, isMacro: true,
1695
+ ...(record.receiver && { receiver: record.receiver, isPathMacro: true }),
1696
+ ...(record.macroExpr && { macroExpr: true }),
1697
+ enclosingFunction: record.enclosingFunction,
1698
+ macroExpanded: true,
1699
+ };
1700
+ ctx.invocations.set(record, invocation);
1701
+ }
1702
+ out.push(invocation);
1703
+ }
1704
+ return out;
1705
+ }
1706
+
1707
+ /** Whether a parsed record lies in a unit of a group read so far (all units, one pass). */
1708
+ function inReadUnit(ctx, record) {
1709
+ if (!ctx.isolated) return ctx.lookups.inUnit(record);
1710
+ if (!ctx.readLookups) {
1711
+ const units = [];
1712
+ for (const group of ctx.groups) if (group.units) units.push(...group.units);
1713
+ units.sort((a, b) => a.start - b.start);
1714
+ ctx.readLookups = unitLookups(units);
1715
+ }
1716
+ return ctx.readLookups.inUnit(record);
1717
+ }
1718
+
1719
+ /**
1720
+ * The calls of the file: parsed records outside the units, each group's
1721
+ * records, the invocations; stable by line. Installed in the calls cache
1722
+ * with the facts the file entry derives from them.
1723
+ */
1724
+ function composeCalls(index, ctx) {
1725
+ readGroups(ctx, ctx.groups.map((group, i) => i));
1726
+ const fileEntry = index.files.get(ctx.filePath);
1727
+ const calls = ctx.original.filter(record => !inReadUnit(ctx, record));
1728
+ for (const group of ctx.groups) for (const record of group.records) calls.push(record);
1729
+ for (const record of invocationRecords(ctx)) calls.push(record);
1730
+ // Shapes of generated calls whose value may be lost (audit-async).
1731
+ const { openCallShape } = require('../languages/rust-value-flow');
1732
+ const shapes = calls.filter(call => call.macroExpansion && call.name && !call.inMacro && !call.valueConsumed)
1733
+ .map(openCallShape);
1734
+ if (shapes.length > 0) {
1735
+ fileEntry.openCalls = [...new Set([...(fileEntry.openCalls || []), ...shapes])].sort(codeUnitCompare);
1736
+ }
1737
+ // Stable by line: records keep document order within a line.
1738
+ calls.sort((a, b) => a.line - b.line);
1739
+ const cached = index.callsCache.get(ctx.filePath);
1740
+ if (cached) {
1741
+ if (index.calleeIndex) index._removeFromCalleeIndex(ctx.filePath, ctx.original);
1742
+ cached.calls = calls;
1743
+ if (index.calleeIndex) index._addToCalleeIndex(ctx.filePath, calls);
1744
+ } else {
1745
+ index.callsCache.set(ctx.filePath, { mtime: fileEntry.mtime, hash: fileEntry.hash, calls });
1746
+ }
1747
+ index.callsCacheDirty = true;
1748
+ return calls;
1749
+ }
1750
+
1751
+ const CALL_NODES = new Set(['call_expression', 'macro_invocation', 'struct_expression', 'generic_function']);
1752
+
1753
+ /** Layout mark of the name token of a call record inside a region. */
1754
+ function nameMarkOf(root, region, record) {
1755
+ const start = record.callStart;
1756
+ const end = record.callEnd;
1757
+ let node = Number.isFinite(start) && Number.isFinite(end) && end > start
1758
+ ? root.descendantForIndex(start, end - 1) : null;
1759
+ while (node && !(CALL_NODES.has(node.type) && node.startIndex === start && node.endIndex === end)) {
1760
+ node = node.parent;
1761
+ }
1762
+ let nameNode = null;
1763
+ if (node?.type === 'macro_invocation') {
1764
+ nameNode = node.childForFieldName('macro');
1765
+ } else if (node) {
1766
+ let fn = node.childForFieldName('function') || node.childForFieldName('name');
1767
+ if (fn?.type === 'generic_function') fn = fn.childForFieldName('function');
1768
+ if (fn?.type === 'field_expression') nameNode = fn.childForFieldName('field');
1769
+ else if (fn?.type === 'scoped_identifier' || fn?.type === 'scoped_type_identifier') nameNode = fn.childForFieldName('name');
1770
+ else nameNode = fn;
1771
+ }
1772
+ if (nameNode && nameNode.startIndex >= region.synthStart && nameNode.endIndex <= region.synthEnd) {
1773
+ const rel = nameNode.type === 'scoped_identifier' || nameNode.type === 'macro_invocation'
1774
+ ? nameNode.endIndex - 1 - region.synthStart : nameNode.startIndex - region.synthStart;
1775
+ return markAt(region.layout.marks, rel, false);
1776
+ }
1777
+ // No node at that span: the first leaf spelling the name.
1778
+ const marks = region.layout.marks;
1779
+ return marks.find(mark => mark.leaf.v === record.name &&
1780
+ mark.s >= start - region.synthStart && mark.e <= end - region.synthStart) || null;
1781
+ }
1782
+
1783
+ /**
1784
+ * The declarations the expansions generate (IR items carrying
1785
+ * `macroExpansion`), and the invocation spans whose recovered declarations
1786
+ * they replace. Pure: computed in a build worker or in place.
1787
+ */
1788
+ function generatedSymbols(filePath, parsed, { regions, mapper, generated }) {
1789
+ const { createFileIR } = require('./ir');
1790
+ const ir = createFileIR({ language: 'rust', file: filePath, parsed, calls: [] });
1791
+ const symbols = [];
1792
+ const seen = new Set();
1793
+ let ordinal = 0;
1794
+ for (const item of ir.symbols) {
1795
+ const origin = generated.get(`${item.startLine}\0${item.name}`);
1796
+ if (!origin) continue;
1797
+ // Declarations a repetition generates identically at one line.
1798
+ const key = `${item.kind}\0${item.name}\0${item.owner || ''}\0${item.startLine}\0${item.traitName || ''}\0` +
1799
+ `${item.params || ''}\0${item.returnType || ''}`;
1800
+ if (seen.has(key)) continue;
1801
+ seen.add(key);
1802
+ const symbol = { ...item };
1803
+ remapOffsets(symbol, mapper);
1804
+ symbol.macroExpansion = macroFact(origin.region, origin.origin);
1805
+ if (origin.nameLine && origin.nameLine !== symbol.startLine) symbol.nameLine = origin.nameLine;
1806
+ symbol.id = `macro:${symbol.startLine}:${symbol.kind}:${symbol.owner || ''}:${symbol.name}:${ordinal++}`;
1807
+ symbols.push(symbol);
1808
+ }
1809
+ return {
1810
+ symbols,
1811
+ spans: regions.map(region => ({
1812
+ startLine: region.startLine, endLine: region.endLine, ancestors: [...region.ancestors],
1813
+ })),
1814
+ };
1815
+ }
1816
+
1817
+ /** Apply generatedSymbols() to the index (main thread). */
1818
+ function applyGeneratedSymbols(index, filePath, result) {
1819
+ const fileEntry = index.files.get(filePath);
1820
+ if (!fileEntry || !result) return;
1821
+ for (const symbol of fileEntry.symbols) {
1822
+ const entries = index.symbols.get(symbol.name);
1823
+ if (!entries) continue;
1824
+ const kept = entries.filter(entry => entry.file !== filePath);
1825
+ if (kept.length > 0) index.symbols.set(symbol.name, kept);
1826
+ else index.symbols.delete(symbol.name);
1827
+ }
1828
+ const previous = fileEntry.symbols;
1829
+ fileEntry.symbols = [];
1830
+ fileEntry.bindings = [];
1831
+ // An invocation's token trees held only what the parser recovered from
1832
+ // them (item bodies): those give way to what the expansion declares.
1833
+ // The declarations enclosing an invocation stay.
1834
+ const spans = result.spans.map(span => ({ ...span, ancestors: new Set(span.ancestors) }));
1835
+ const insideSpan = symbol => spans.some(span => symbol.startLine >= span.startLine &&
1836
+ symbol.endLine <= span.endLine && !span.ancestors.has(`${symbol.startLine}\0${symbol.name}`));
1837
+ for (const symbol of previous) {
1838
+ if (insideSpan(symbol)) continue;
1839
+ fileEntry.symbols.push(symbol);
1840
+ if (!index.symbols.has(symbol.name)) index.symbols.set(symbol.name, []);
1841
+ index.symbols.get(symbol.name).push(symbol);
1842
+ if (symbol.memberAssigned || symbol.bodyScopedName || symbol.exportedAlias || symbol.type === 'impl') continue;
1843
+ fileEntry.bindings.push({
1844
+ id: symbol.bindingId, name: symbol.name, type: symbol.type, startLine: symbol.startLine,
1845
+ });
1846
+ }
1847
+ for (const item of result.symbols) {
1848
+ const symbol = addIRSymbol(fileEntry, item, index.symbols);
1849
+ // A generated trait impl for a type declared in this file joins the
1850
+ // type's `implements` (the parser does this for impls it sees).
1851
+ if (symbol.type !== 'impl' || !symbol.traitName) continue;
1852
+ const owner = fileEntry.symbols.find(candidate => (candidate.type === 'struct' || candidate.type === 'enum') &&
1853
+ candidate.name === symbol.name && !candidate.macroExpansion);
1854
+ if (owner && !(owner.implements || []).includes(symbol.traitName)) {
1855
+ owner.implements = [...(owner.implements || []), symbol.traitName];
1856
+ }
1857
+ }
1858
+ }
1859
+
1860
+ /**
1861
+ * Names the calls of the expansions can bear (callee position, method,
1862
+ * macro, struct literal): the callee index lists the file under them until
1863
+ * its calls are expanded.
1864
+ */
1865
+ function callableNames(regions) {
1866
+ const names = new Set();
1867
+ for (const region of regions) {
1868
+ const marks = region.layout.marks;
1869
+ for (let i = 0; i < marks.length - 1; i++) {
1870
+ const next = marks[i + 1].leaf.v;
1871
+ if (!(next === '(' || next === '!' || next === '{' ||
1872
+ (next === '::' && marks[i + 2]?.leaf.v === '<'))) continue;
1873
+ const leaf = marks[i].leaf;
1874
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(leaf.v) || RUST_KEYWORDS.has(leaf.v)) continue;
1875
+ names.add(leaf.v);
1876
+ }
1877
+ }
1878
+ return [...names].sort(codeUnitCompare);
1879
+ }
1880
+
1881
+ const WORD = /[\p{L}\p{N}_]+/gu;
1882
+ const WHOLE_WORD = /^[\p{L}\p{N}_]+$/u;
1883
+
1884
+ // Tokens whose text is never an identifier: literals and comments.
1885
+ const NON_CODE_TOKENS = ['string_literal', 'raw_string_literal', 'char_literal', 'line_comment', 'block_comment'];
1886
+
1887
+ /**
1888
+ * The words (identifier-like runs) of the text the calls pass analyzes for
1889
+ * these regions: the kept source around them (units and their headers) and
1890
+ * every expansion's tokens. A call record derived from that text can bear no
1891
+ * other name, so a file whose calls are not derived yet needs deriving only
1892
+ * for a name among them (fix #375). A record's name and written receiver are
1893
+ * identifier tokens, so the text of literals and comments is left out (fix
1894
+ * #405: `"--count"` and `// the count` made a `count` query derive four
1895
+ * test files); kept ranges are whole items and headers, which the calls
1896
+ * pass lexes as the file's own parse did. A file with parse errors is read
1897
+ * whole. Space-separated, sorted.
1898
+ */
1899
+ function expansionWords(content, regions, nonCode = [], outputs = regions.map(outputWordsOf)) {
1900
+ const words = new Set();
1901
+ scanCodeWords(content, analysisUnits(regions, 'calls').keep, nonCode, words);
1902
+ for (const output of outputs) for (const word of output) words.add(word);
1903
+ return [...words].sort(codeUnitCompare).join(' ');
1904
+ }
1905
+
1906
+ /** Words of `ranges` (sorted, disjoint) of the source, literals and comments (`nonCode`, sorted) left out. */
1907
+ function scanCodeWords(content, ranges, nonCode, words) {
1908
+ const scan = (start, end) => {
1909
+ if (end <= start) return;
1910
+ for (const word of content.slice(start, end).match(WORD) || []) words.add(word);
1911
+ };
1912
+ let next = 0;
1913
+ for (const [start, end] of ranges) {
1914
+ let at = start;
1915
+ while (next < nonCode.length && nonCode[next].endIndex <= at) next++;
1916
+ for (let i = next; i < nonCode.length; i++) {
1917
+ const token = nonCode[i];
1918
+ if (token.startIndex >= end) break;
1919
+ scan(at, Math.min(token.startIndex, end));
1920
+ at = Math.max(at, token.endIndex);
1921
+ }
1922
+ scan(at, end);
1923
+ }
1924
+ }
1925
+
1926
+ /** The identifier tokens an expansion outputs (a non-ASCII identifier is a 'punct' leaf), once each. */
1927
+ function outputWordsOf(region) {
1928
+ const words = new Set();
1929
+ for (const mark of region.layout.marks) {
1930
+ const leaf = mark.leaf;
1931
+ if (leaf.k === 'ident' || (leaf.k === 'punct' && WHOLE_WORD.test(leaf.v))) words.add(leaf.v);
1932
+ }
1933
+ return [...words];
1934
+ }
1935
+
1936
+ /**
1937
+ * The groups of units the calls pass reads apart (fix #405b), each as
1938
+ * [startLine, endLine, expanded length, words]: the identifiers its
1939
+ * expansions output (with those of any expansion in its kept headers) and
1940
+ * the code of its kept headers outside its units (`Self { .. }` names the
1941
+ * impl's type). A record a group derives in the units' own code is written
1942
+ * there, as the file's parsed record on that line is: the calls pass
1943
+ * consults those for the rest (rustPendingRecords). Expanded length: the
1944
+ * text the group's own expansions add, for the cost of reading it.
1945
+ */
1946
+ function groupRecords(content, regions, nonCode, outputs) {
1947
+ const { units } = analysisUnits(regions, 'calls');
1948
+ const groups = groupsOf(units);
1949
+ const lookups = unitLookups(units);
1950
+ const members = groups.map(() => []);
1951
+ const groupOf = regions.map((region, i) => {
1952
+ const group = lookups.unitAtOffset(region.start).group;
1953
+ members[group].push(i);
1954
+ return group;
1955
+ });
1956
+ // Expansions are disjoint and sorted: the first one ending after `at`.
1957
+ const firstEndingAfter = at => {
1958
+ let lo = 0;
1959
+ let hi = regions.length;
1960
+ while (lo < hi) {
1961
+ const mid = (lo + hi) >> 1;
1962
+ if (regions[mid].end <= at) lo = mid + 1; else hi = mid;
1963
+ }
1964
+ return lo;
1965
+ };
1966
+ return groups.map((group, gi) => {
1967
+ const keep = analysisUnits(members[gi].map(i => regions[i]), 'calls').keep;
1968
+ const words = new Set();
1969
+ let length = 0;
1970
+ for (const i of members[gi]) {
1971
+ length += regions[i].layout.text.length;
1972
+ for (const word of outputs[i]) words.add(word);
1973
+ }
1974
+ // Expansions inside its kept headers (`impl T for m!() {..}`).
1975
+ for (const [start, end] of keep) {
1976
+ for (let i = firstEndingAfter(start); i < regions.length && regions[i].start < end; i++) {
1977
+ if (groupOf[i] !== gi) for (const word of outputs[i]) words.add(word);
1978
+ }
1979
+ }
1980
+ // Header code: the kept ranges outside the group's units.
1981
+ const headers = [];
1982
+ for (const [start, end] of keep) {
1983
+ let at = start;
1984
+ for (const unit of group.units) {
1985
+ if (unit.end <= at || unit.start >= end) continue;
1986
+ if (unit.start > at) headers.push([at, unit.start]);
1987
+ at = Math.max(at, unit.end);
1988
+ }
1989
+ if (at < end) headers.push([at, end]);
1990
+ }
1991
+ scanCodeWords(content, headers, nonCode, words);
1992
+ // Insertion order is the expansions' and the text's: deterministic.
1993
+ return [group.startLine, group.endLine, length, [...words].join(' ')];
1994
+ });
1995
+ }
1996
+
1997
+ /** Whether the derived calls of a file pending derivation can bear `name`. */
1998
+ const pendingWordSets = new WeakMap();
1999
+ function rustPendingMayBear(fileEntry, name) {
2000
+ const rec = fileEntry?.rustMacroExpansion;
2001
+ if (!rec?.callsPending) return false;
2002
+ if (typeof rec.words !== 'string') return true;
2003
+ let words = pendingWordSets.get(rec);
2004
+ if (!words) pendingWordSets.set(rec, words = new Set(rec.words.split(' ')));
2005
+ return hasWord(words, name);
2006
+ }
2007
+
2008
+ /**
2009
+ * Whether a recorded group's word list (space-separated) holds `name`, as
2010
+ * hasWord does for a set (fix #405b: one name is asked per group, so the
2011
+ * list is searched rather than split).
2012
+ */
2013
+ function groupHasWord(list, name) {
2014
+ const text = String(name);
2015
+ const bare = text.replace(/^r#/, '');
2016
+ return listHasWord(list, text) || (bare !== text && listHasWord(list, bare)) || listHasWord(list, `r#${text}`);
2017
+ }
2018
+
2019
+ function listHasWord(list, word) {
2020
+ if (!word) return false;
2021
+ for (let at = list.indexOf(word); at >= 0; at = list.indexOf(word, at + 1)) {
2022
+ const end = at + word.length;
2023
+ if ((at === 0 || list.charCodeAt(at - 1) === 32) && (end === list.length || list.charCodeAt(end) === 32)) return true;
2024
+ }
2025
+ return false;
2026
+ }
2027
+
2028
+ const pendingFileLists = new WeakMap(); // operation content cache -> pending file paths
2029
+
2030
+ /**
2031
+ * Files whose macro expansions' calls are not derived yet and whose
2032
+ * expanded text can hold one of `names` (fix #405). The callee index lists
2033
+ * such a file under its parsed records' names and its expansions'
2034
+ * callee-position names, but a derived record can bear any word of the
2035
+ * expanded text (`self.view` passed as a value, `Self { .. }` named by its
2036
+ * impl): a query visits these files too, so the files it reads do not
2037
+ * depend on which files earlier queries derived.
2038
+ */
2039
+ function rustPendingFilesBearing(index, names) {
2040
+ const scope = index._opContentCache;
2041
+ let files = scope ? pendingFileLists.get(scope) : undefined;
2042
+ if (!files) {
2043
+ files = [];
2044
+ for (const [filePath, fileEntry] of index.files) {
2045
+ if (fileEntry.rustMacroExpansion?.callsPending) files.push(filePath);
2046
+ }
2047
+ if (scope) pendingFileLists.set(scope, files);
2048
+ }
2049
+ const bearing = [];
2050
+ for (const filePath of files) {
2051
+ const fileEntry = index.files.get(filePath);
2052
+ if (names.some(name => rustPendingMayBear(fileEntry, name))) bearing.push(filePath);
2053
+ }
2054
+ return bearing;
2055
+ }
2056
+
2057
+ function tokenText(tokens) {
2058
+ return flatten(tokens).map(leaf => leaf.v).join(' ');
2059
+ }
2060
+
2061
+ // ── Build pass ────────────────────────────────────────────────────────────
2062
+
2063
+ /** Project macro names a file's call records invoke. */
2064
+ function invokedMacroNames(index, filePath, inventory) {
2065
+ const calls = index.callsCache.get(filePath)?.calls;
2066
+ if (!Array.isArray(calls)) return [];
2067
+ const names = new Set();
2068
+ for (const call of calls) {
2069
+ if (call.isMacro && !call.macroExpansion && inventory.has(call.name)) names.add(call.name);
2070
+ }
2071
+ return [...names].sort(codeUnitCompare);
2072
+ }
2073
+
2074
+ function signatureFor(state, filePath, names) {
2075
+ const index = state.index;
2076
+ const hash = crypto.createHash('md5');
2077
+ hash.update(index.files.get(filePath)?.hash || '');
2078
+ for (const name of names) {
2079
+ for (const def of state.inventory.get(name) || []) {
2080
+ const entry = index.files.get(def.file);
2081
+ hash.update(`\n${name}\0${def.relativePath}\0${def.startLine}\0${def.endLine}\0${entry?.hash || ''}`);
2082
+ }
2083
+ }
2084
+ // Module ancestry decides textual scope; crate roots decide path scope.
2085
+ const parents = moduleParents(state);
2086
+ let current = filePath;
2087
+ const seen = new Set();
2088
+ while (parents.has(current) && !seen.has(current)) {
2089
+ seen.add(current);
2090
+ const link = parents.get(current);
2091
+ current = link.file;
2092
+ hash.update(`\n^${path.relative(index.root, current)}\0${link.line}\0${link.macroUse ? 1 : 0}\0` +
2093
+ `${index.files.get(current)?.hash || ''}`);
2094
+ }
2095
+ return hash.digest('hex');
2096
+ }
2097
+
2098
+ /**
2099
+ * Build-time pass, phase 1 (before the import graph): Rust files whose
2100
+ * persisted expansion is missing or stale. An expanded file is re-parsed
2101
+ * (`reindex`) so phase 2 starts from its parser facts.
2102
+ */
2103
+ function planRustMacroExpansion(index, { reindex = null, files = null } = {}) {
2104
+ const rustFiles = [];
2105
+ for (const [file, entry] of index.files) {
2106
+ if (isRust(entry) && (!files || files.includes(file))) rustFiles.push(file);
2107
+ }
2108
+ if (rustFiles.length === 0) return [];
2109
+ const state = createState(index);
2110
+ const pending = [];
2111
+ for (const file of rustFiles) {
2112
+ const entry = index.files.get(file);
2113
+ const rec = entry.rustMacroExpansion;
2114
+ const names = state.inventory.size > 0 ? invokedMacroNames(index, file, state.inventory) : [];
2115
+ if (names.length === 0 && !rec) continue;
2116
+ if (rec && names.length > 0 && names.join(',') === (rec.callNames || []).join(',') &&
2117
+ signatureFor(state, file, mergedNames(names, rec.nested)) === rec.signature) continue;
2118
+ pending.push(file);
2119
+ }
2120
+ for (const file of pending) {
2121
+ const rec = index.files.get(file)?.rustMacroExpansion;
2122
+ if (rec?.expanded && reindex) reindex(file);
2123
+ const entry = index.files.get(file);
2124
+ if (entry) delete entry.rustMacroExpansion;
2125
+ }
2126
+ return pending;
2127
+ }
2128
+
2129
+ function mergedNames(names, nested) {
2130
+ return [...new Set([...names, ...(nested || [])])].sort(codeUnitCompare);
2131
+ }
2132
+
2133
+ /** Planned files that invoke a project macro (from their call records). */
2134
+ function invokingFiles(index, pending, inventory, names) {
2135
+ return pending.filter(file => {
2136
+ if (!index.files.has(file)) return false;
2137
+ const invoked = invokedMacroNames(index, file, inventory);
2138
+ names.set(file, invoked);
2139
+ return invoked.length > 0;
2140
+ });
2141
+ }
2142
+
2143
+ /**
2144
+ * Build-time pass, phase 2a (fix #375): start expanding the planned files in
2145
+ * worker threads while the caller builds the import graph. The workers
2146
+ * resolve module ancestry from the `mod` declarations alone (the only import
2147
+ * facts expansion reads), so their records equal an in-place expansion after
2148
+ * the graph. Returns a handle for applyRustMacroExpansion (null when there is
2149
+ * nothing to expand; `results` null when the files expand in place).
2150
+ */
2151
+ function startRustMacroExpansion(index, pending, options = {}) {
2152
+ if (!pending || pending.length === 0) return null;
2153
+ const inventory = macroInventory(index);
2154
+ if (inventory.size === 0) return { inventory, files: [], results: null };
2155
+ const names = new Map();
2156
+ const files = invokingFiles(index, pending, inventory, names);
2157
+ let bytes = 0;
2158
+ for (const file of files) bytes += index.files.get(file)?.size || 0;
2159
+ const run = files.length >= PARALLEL_MIN_FILES && bytes >= PARALLEL_MIN_BYTES
2160
+ ? startWorkers(index, files, { ...options, inventory }) : null;
2161
+ return { inventory, files, names, results: run };
2162
+ }
2163
+
2164
+ /**
2165
+ * Build-time pass, phase 2b (after the import graph resolved `mod`
2166
+ * declarations): apply the expansions of the planned files, expanding in
2167
+ * place what no worker returned. Returns false when nothing expanded, else
2168
+ * { files, names }: the files whose symbols were replaced and the symbol
2169
+ * names involved.
2170
+ */
2171
+ function applyRustMacroExpansion(index, pending, options = {}, started = undefined) {
2172
+ const run = started === undefined ? startRustMacroExpansion(index, pending, options) : started;
2173
+ if (!run) return false;
2174
+ let results;
2175
+ try {
2176
+ results = run.results ? run.results.collect() : null;
2177
+ } finally {
2178
+ run.results?.dispose();
2179
+ }
2180
+ if (run.files.length === 0) return false;
2181
+ const state = createState(index, run.inventory);
2182
+ // What the expansions changed: the files whose symbols were replaced and
2183
+ // the symbol names involved (the caller re-canonicalizes only those).
2184
+ let touched = null;
2185
+ for (const file of run.files) {
2186
+ const entry = index.files.get(file);
2187
+ // Symbols-mode expansion never changes call records.
2188
+ const callNames = run.names?.get(file) || invokedMacroNames(index, file, state.inventory);
2189
+ let rec = results?.get(file);
2190
+ if (!rec) {
2191
+ rec = expandFile(state, file) || { sites: 0, blind: [], used: new Map(), expanded: false };
2192
+ rec = { ...rec, usedNames: [...rec.used.values()].map(def => def.name) };
2193
+ }
2194
+ if (rec.symbols) {
2195
+ touched ||= { files: new Set(), names: new Set() };
2196
+ touched.files.add(file);
2197
+ for (const symbol of entry.symbols) touched.names.add(symbol.name);
2198
+ applyGeneratedSymbols(index, file, rec.symbols);
2199
+ for (const symbol of entry.symbols) touched.names.add(symbol.name);
2200
+ }
2201
+ if (rec.expanded && !touched) touched = { files: new Set(), names: new Set() };
2202
+ const nested = [...new Set(rec.usedNames || [])]
2203
+ .filter(name => !callNames.includes(name)).sort(codeUnitCompare);
2204
+ entry.rustMacroExpansion = {
2205
+ signature: signatureFor(state, file, mergedNames(callNames, nested)),
2206
+ callNames,
2207
+ ...(nested.length > 0 && { nested }),
2208
+ sites: rec.sites,
2209
+ ...(rec.blind.length > 0 && { blind: rec.blind.sort((a, b) => a.line - b.line ||
2210
+ codeUnitCompare(a.macro, b.macro) || codeUnitCompare(a.reason, b.reason)) }),
2211
+ ...(rec.expanded && { expanded: true }),
2212
+ ...(rec.callsPending && {
2213
+ callsPending: true, calleeNames: rec.calleeNames || [], words: rec.words, expansions: rec.expansions,
2214
+ ...(rec.groups && { groups: rec.groups }),
2215
+ }),
2216
+ };
2217
+ }
2218
+ return touched || false;
2219
+ }
2220
+
2221
+ // Parse workers kept for expansion pay for themselves from a few files of
2222
+ // expansion work; the job is posted while the import graph builds.
2223
+ const PARALLEL_MIN_FILES = 2;
2224
+ const PARALLEL_MIN_BYTES = 32 * 1024;
2225
+ // Parse workers a parallel build keeps for expansion (fix #388). Their parser
2226
+ // code is already optimized, so a few of them finish the largest invoking
2227
+ // files about as fast as more fresh threads would, at a fraction of the CPU
2228
+ // (measured on ripgrep, tokio, rayon, axum, hyper, cargo, clap).
2229
+ const EXPANSION_WORKERS = 3;
2230
+
2231
+ /**
2232
+ * What a worker needs to expand `workload` without the index: the module
2233
+ * tree (every Rust file's `mod` declarations, resolved, and module symbols),
2234
+ * the macro definitions (their text is read from disk), and for the files
2235
+ * to expand their import facts, impl/trait spans and macro invocation
2236
+ * records. Also every name a project callable bears.
2237
+ */
2238
+ function workerSnapshot(index, workload, inventory) {
2239
+ const expanding = new Set(workload);
2240
+ // Invocations resolve at the invoking file, and nested ones also at the
2241
+ // defining file.
2242
+ const resolving = new Set(workload);
2243
+ for (const defs of inventory.values()) for (const def of defs) resolving.add(def.file);
2244
+ const files = [];
2245
+ const macros = [];
2246
+ for (const [file, entry] of index.files) {
2247
+ if (!isRust(entry)) continue;
2248
+ const full = expanding.has(file);
2249
+ const resolves = resolving.has(file);
2250
+ // Module ancestry reads `mod` declarations (and the module symbols'
2251
+ // attributes); resolution at a file also reads its glob imports and
2252
+ // the `use` bindings that can name a macro; expansion reads the
2253
+ // invoking file's impl/trait spans and project macro invocations.
2254
+ const record = {
2255
+ path: file, relativePath: entry.relativePath, language: 'rust',
2256
+ importDetails: (entry.importDetails || []).filter(detail => detail.type === 'mod' ||
2257
+ (resolves && detail.type === 'use-glob')),
2258
+ symbols: (entry.symbols || []).filter(symbol => symbol.type === 'module' ||
2259
+ (full && (symbol.type === 'impl' || symbol.type === 'trait')))
2260
+ .map(symbol => ({
2261
+ name: symbol.name, type: symbol.type, startLine: symbol.startLine, endLine: symbol.endLine,
2262
+ modifiers: symbol.modifiers || [],
2263
+ })),
2264
+ };
2265
+ if (resolves) {
2266
+ record.importBindings = (entry.importBindings || [])
2267
+ .filter(binding => inventory.has(binding.alias || binding.name));
2268
+ }
2269
+ if (full) {
2270
+ record.calls = (index.callsCache.get(file)?.calls || [])
2271
+ .filter(call => call.isMacro && !call.macroExpansion && inventory.has(call.name))
2272
+ .map(call => ({
2273
+ name: call.name, line: call.line, callStart: call.callStart, callEnd: call.callEnd,
2274
+ isMacro: true,
2275
+ ...(call.inMacro && { inMacro: true }),
2276
+ ...(call.macroExpr && { macroExpr: true }),
2277
+ ...(call.isPathMacro && { isPathMacro: true, receiver: call.receiver }),
2278
+ ...(call.enclosingFunction && { enclosingFunction: { name: call.enclosingFunction.name } }),
2279
+ }));
2280
+ }
2281
+ files.push(record);
2282
+ for (const symbol of entry.symbols || []) {
2283
+ if (symbol.type === 'macro') {
2284
+ macros.push({ name: symbol.name, type: 'macro', file, relativePath: symbol.relativePath,
2285
+ startLine: symbol.startLine, endLine: symbol.endLine, modifiers: symbol.modifiers || [],
2286
+ ...(symbol.macroScope && { macroScope: symbol.macroScope }) });
2287
+ }
2288
+ }
2289
+ }
2290
+ // Callee-position macros expand where an argument word names a project
2291
+ // callable: the workers see every such name.
2292
+ const callable = [];
2293
+ for (const [name, defs] of index.symbols) {
2294
+ if (defs.some(def => !NON_CALLABLE_DEF_TYPES.has(def.type))) callable.push(name);
2295
+ }
2296
+ return {
2297
+ root: index.root, files, macros, callableNames: callable,
2298
+ // `mod` declarations resolve in the worker, with this build's view
2299
+ // of the workspace manifests.
2300
+ config: { aliases: index.config?.aliases, includePaths: index.config?.includePaths },
2301
+ cargoManifests: Array.isArray(index.cargoManifests) ? index.cargoManifests : null,
2302
+ };
2303
+ }
2304
+
2305
+ /** An index-shaped view of a worker snapshot (what createState reads). */
2306
+ function snapshotIndex(snapshot) {
2307
+ const { rustModuleDeclarationsResolved, seedWorkspaceManifests } = require('./imports');
2308
+ if (snapshot.cargoManifests) seedWorkspaceManifests(snapshot.root, snapshot.cargoManifests);
2309
+ const files = new Map();
2310
+ const callsCache = new Map();
2311
+ const view = { root: snapshot.root, config: snapshot.config || {}, files };
2312
+ for (const file of snapshot.files) {
2313
+ files.set(file.path, file);
2314
+ if (file.calls) callsCache.set(file.path, { calls: file.calls });
2315
+ let resolved = null;
2316
+ Object.defineProperty(file, 'moduleResolved', {
2317
+ get: () => (resolved ||= rustModuleDeclarationsResolved(view, file.path, file)),
2318
+ });
2319
+ }
2320
+ const symbols = new Map();
2321
+ for (const def of snapshot.macros) {
2322
+ if (!symbols.has(def.name)) symbols.set(def.name, []);
2323
+ symbols.get(def.name).push(def);
2324
+ }
2325
+ let callable = null;
2326
+ return {
2327
+ root: snapshot.root, files, callsCache, symbols,
2328
+ callableNames: {
2329
+ has: word => (callable ||= new Set(snapshot.callableNames || [])).has(word),
2330
+ },
2331
+ _readFile: file => fs.readFileSync(file, 'utf-8'),
2332
+ };
2333
+ }
2334
+
2335
+ /**
2336
+ * Worker-side: expand the files of a shared queue (`files`, next index at
2337
+ * `signal[queueIndex]`), posting each record to `port`; `signal[workerIndex]`
2338
+ * is set when this worker is done.
2339
+ */
2340
+ function runExpansionQueue({ snapshot, files, signal, workerIndex, queueIndex }, port) {
2341
+ const signalArray = new Int32Array(signal);
2342
+ try {
2343
+ const state = createState(snapshotIndex(JSON.parse(Buffer.from(snapshot).toString('utf8'))));
2344
+ for (;;) {
2345
+ const next = Atomics.add(signalArray, queueIndex, 1);
2346
+ if (next >= files.length) break;
2347
+ const file = files[next];
2348
+ let result;
2349
+ try {
2350
+ result = { file, record: expandForWorker(state, file) };
2351
+ } catch (error) {
2352
+ result = { file, error: error.message };
2353
+ }
2354
+ port.postMessage([result]);
2355
+ }
2356
+ } finally {
2357
+ Atomics.store(signalArray, workerIndex, 1);
2358
+ Atomics.notify(signalArray, workerIndex);
2359
+ }
2360
+ }
2361
+
2362
+ /** Worker-side: symbols-mode expansion of one file, as plain records. */
2363
+ function expandForWorker(state, filePath) {
2364
+ const rec = expandFile(state, filePath) || { sites: 0, blind: [], used: new Map(), expanded: false };
2365
+ return {
2366
+ sites: rec.sites, blind: rec.blind, expanded: rec.expanded,
2367
+ usedNames: [...rec.used.values()].map(def => def.name),
2368
+ ...(rec.symbols && { symbols: rec.symbols }),
2369
+ ...(rec.callsPending && {
2370
+ callsPending: true, calleeNames: rec.calleeNames, words: rec.words, expansions: rec.expansions,
2371
+ ...(rec.groups && { groups: rec.groups }),
2372
+ }),
2373
+ };
2374
+ }
2375
+
2376
+ /**
2377
+ * Start expanding `files` in the parse workers the build kept for it
2378
+ * (largest first from a shared queue). Returns { collect(), dispose() }, or
2379
+ * null without kept workers (a sequential build expands in place, where the
2380
+ * parser is already optimized); collect() blocks until the workers finish
2381
+ * and returns Map<file, record> (files a worker failed on are expanded in
2382
+ * place by the caller).
2383
+ */
2384
+ function startWorkers(index, files, options = {}) {
2385
+ const pool = options.pool?.available.length > 0 ? options.pool : null;
2386
+ if (!pool) return null;
2387
+ const { receiveMessageOnPort } = require('worker_threads');
2388
+ const count = Math.min(pool.available.length, EXPANSION_WORKERS, Math.ceil(files.length / 2));
2389
+ if (count < 1) return null;
2390
+ const ordered = [...files].sort((a, b) => (index.files.get(b)?.size || 0) - (index.files.get(a)?.size || 0) ||
2391
+ codeUnitCompare(a, b));
2392
+ // One serialization shared by every worker (a structured clone per
2393
+ // worker cost more than the expansion on large workspaces).
2394
+ const encoded = Buffer.from(JSON.stringify(workerSnapshot(index, files, options.inventory)), 'utf8');
2395
+ const snapshot = new SharedArrayBuffer(encoded.length);
2396
+ encoded.copy(Buffer.from(snapshot));
2397
+ const sab = new SharedArrayBuffer(4 * (count + 1));
2398
+ const signal = new Int32Array(sab);
2399
+ const ports = [];
2400
+ for (let i = 0; i < count; i++) {
2401
+ const worker = pool.available[i];
2402
+ ports.push(pool.ports[worker]);
2403
+ pool.post(worker, { snapshot, files: ordered, signal: sab, workerIndex: i, queueIndex: count });
2404
+ }
2405
+ // Kept workers this job does not need end now.
2406
+ for (const worker of pool.available.slice(count)) pool.retire(worker);
2407
+ // The kept workers belong to the build, which ends them.
2408
+ const dispose = () => {};
2409
+ const collect = () => {
2410
+ const deadline = Date.now() + 300000;
2411
+ let timedOut = false;
2412
+ for (let i = 0; i < count; i++) {
2413
+ while (Atomics.load(signal, i) === 0) {
2414
+ const remaining = deadline - Date.now();
2415
+ if (remaining <= 0) { timedOut = true; break; }
2416
+ Atomics.wait(signal, i, 0, Math.min(remaining, 5000));
2417
+ }
2418
+ if (timedOut) break;
2419
+ }
2420
+ const results = new Map();
2421
+ for (const port of ports) {
2422
+ for (let msg = receiveMessageOnPort(port); msg; msg = receiveMessageOnPort(port)) {
2423
+ for (const result of msg.message) {
2424
+ if (!result.error) results.set(result.file, result.record);
2425
+ }
2426
+ }
2427
+ }
2428
+ return results;
2429
+ };
2430
+ return { collect, dispose, usesPool: true, workerCount: count };
2431
+ }
2432
+
2433
+ /**
2434
+ * Call records of a Rust file whose expansions' calls are not derived yet
2435
+ * (the build derives generated declarations; calls follow on first use and
2436
+ * are then persisted). `materialize: false` (callee index upkeep) returns the
2437
+ * parsed records plus name-only entries for every name the expansions can
2438
+ * call, without expanding.
2439
+ */
2440
+ function rustExpandedCalls(index, filePath, rawCalls, { materialize = true } = {}) {
2441
+ const rec = index.files.get(filePath)?.rustMacroExpansion;
2442
+ if (!rec?.callsPending || !Array.isArray(rawCalls)) return rawCalls;
2443
+ const hints = (rec.calleeNames || []).map(name => ({ name, line: 0, calleeHint: true }));
2444
+ if (!materialize) return hints.length > 0 ? rawCalls.concat(hints) : rawCalls;
2445
+ const ctx = pendingContext(index, filePath, { whole: true });
2446
+ if (index.calleeIndex && hints.length > 0) index._removeFromCalleeIndex(filePath, hints);
2447
+ rec.callsPending = false;
2448
+ try {
2449
+ if (ctx) composeCalls(index, ctx);
2450
+ } catch (error) {
2451
+ rec.callsPending = true;
2452
+ throw error;
2453
+ }
2454
+ index._rustMacroState.contexts.delete(filePath);
2455
+ delete rec.words;
2456
+ delete rec.groups;
2457
+ delete rec.expansions;
2458
+ index.macroExpansionDirty = true;
2459
+ return index.callsCache.get(filePath)?.calls || rawCalls;
2460
+ }
2461
+
2462
+ // Derivation contexts whose trees are kept (fix #405); a context dropped
2463
+ // earlier keeps the records of the groups it read.
2464
+ const LIVE_CONTEXTS = 3;
2465
+
2466
+ /**
2467
+ * The calls-pass context of a file whose expansion calls are pending, or
2468
+ * null when nothing of it expands (its parsed records stand).
2469
+ */
2470
+ function pendingContext(index, filePath, { whole = false } = {}) {
2471
+ if (!index._rustMacroState) index._rustMacroState = createState(index);
2472
+ const state = index._rustMacroState;
2473
+ const contexts = state.contexts;
2474
+ let ctx = contexts.get(filePath);
2475
+ if (ctx === null) return null;
2476
+ if (ctx && ctx.live) {
2477
+ contexts.delete(filePath);
2478
+ contexts.set(filePath, ctx);
2479
+ return ctx;
2480
+ }
2481
+ // A file derived whole before any part of it was read is read in one
2482
+ // pass (fix #405b): the groups apart cost the same expansions and parse
2483
+ // plus a second plan of the file's invocations.
2484
+ const fresh = whole && ctx === undefined
2485
+ ? expandFile(state, filePath, 'calls')?.context || null
2486
+ : buildContext(state, filePath);
2487
+ if (fresh && ctx) {
2488
+ // Rebuilt: the groups the dropped context read keep their records.
2489
+ if (fresh.groups.length !== ctx.groups.length || !!fresh.isolated !== !!ctx.isolated) {
2490
+ throw new Error(`Rust macro expansion context changed: ${filePath}`);
2491
+ }
2492
+ for (let i = 0; i < fresh.groups.length; i++) {
2493
+ const kept = ctx.groups[i];
2494
+ if (!kept.records) continue;
2495
+ Object.assign(fresh.groups[i], { records: kept.records, units: kept.units, starts: kept.starts });
2496
+ }
2497
+ fresh.invocations = ctx.invocations;
2498
+ fresh.seen = ctx.seen;
2499
+ }
2500
+ contexts.delete(filePath);
2501
+ contexts.set(filePath, fresh);
2502
+ if (fresh) {
2503
+ let live = 0;
2504
+ for (const [file, other] of [...contexts].reverse()) {
2505
+ if (!other?.live) continue;
2506
+ if (++live <= LIVE_CONTEXTS || file === filePath) continue;
2507
+ contexts.set(file, releaseContext(other));
2508
+ }
2509
+ }
2510
+ return fresh;
2511
+ }
2512
+
2513
+ /**
2514
+ * A file's calls-pass context: its groups read apart when the build's
2515
+ * record of its expansions still matches its invocations, else one pass
2516
+ * over the whole file. null when nothing of it expands.
2517
+ */
2518
+ function buildContext(state, filePath) {
2519
+ const rec = state.index.files.get(filePath)?.rustMacroExpansion;
2520
+ // Expansions the build could not parse in context are read in one pass
2521
+ // (whose retries can also drop a neighbor).
2522
+ if (Array.isArray(rec?.expansions) && !(rec.blind || []).some(site => site.reason === 'parse-error')) {
2523
+ const isolated = isolatedContext(state, filePath, rec.expansions);
2524
+ if (isolated !== undefined) return isolated;
2525
+ }
2526
+ return expandFile(state, filePath, 'calls')?.context || null;
2527
+ }
2528
+
2529
+ /** What a dropped context keeps: the records it read. */
2530
+ function releaseContext(ctx) {
2531
+ return {
2532
+ isolated: ctx.isolated,
2533
+ groups: ctx.groups.map(group => ({ records: group.records, units: group.units, starts: group.starts })),
2534
+ invocations: ctx.invocations,
2535
+ seen: ctx.seen,
2536
+ live: false,
2537
+ };
2538
+ }
2539
+
2540
+ /**
2541
+ * A context that reads each group apart (fix #405). The build recorded, for
2542
+ * every invocation it expanded, its hygiene start and its expanded length:
2543
+ * a group expands alone with the numbering and the offsets of a pass over
2544
+ * the whole file, every other expansion standing in as a blank of its
2545
+ * length. undefined when that record no longer matches the file's
2546
+ * invocations or the file has parse errors (one pass reads it), null when
2547
+ * nothing of it expands.
2548
+ */
2549
+ function isolatedContext(state, filePath, expansions) {
2550
+ const index = state.index;
2551
+ const content = readContent(state, filePath);
2552
+ if (!index.files.get(filePath) || content === null) return undefined;
2553
+ if (!fileMayExpand(state, filePath)) return null;
2554
+ const tree = safeParse(state.parser, content, undefined, PARSE_OPTIONS);
2555
+ if (tree.rootNode.hasError) return undefined;
2556
+ const plan = plannedInvocations(state, filePath, content, tree, { blind: [] });
2557
+ if (plan.length * 3 !== expansions.length) return undefined;
2558
+ const slots = [];
2559
+ for (let i = 0; i < plan.length; i++) {
2560
+ const item = plan[i];
2561
+ if (item.node.startIndex !== expansions[3 * i]) return undefined;
2562
+ item.hygiene = expansions[3 * i + 1];
2563
+ item.length = expansions[3 * i + 2];
2564
+ if (item.length < 0) continue;
2565
+ slots.push({
2566
+ item, node: item.node, start: item.node.startIndex, end: item.node.endIndex,
2567
+ startLine: item.line, endLine: item.node.endPosition.row + 1,
2568
+ });
2569
+ }
2570
+ if (slots.length === 0) return null;
2571
+ slots.sort((a, b) => a.start - b.start);
2572
+ const { units } = analysisUnits(slots, 'calls');
2573
+ const lookups = unitLookups(units);
2574
+ const groups = groupsOf(units);
2575
+ for (const group of groups) group.slots = [];
2576
+ for (const slot of slots) groups[lookups.unitAtOffset(slot.start).group].slots.push(slot);
2577
+ return {
2578
+ isolated: true, filePath, content, tree, slots, groups,
2579
+ original: index.callsCache.get(filePath)?.calls || [],
2580
+ seen: new Set(), invocations: new Map(), parser: state.parser, state, live: true,
2581
+ };
2582
+ }
2583
+
2584
+ /** Units sharing a line form one group; units get their group's index. */
2585
+ function groupsOf(units) {
2586
+ const groups = [];
2587
+ for (const unit of units) {
2588
+ const last = groups[groups.length - 1];
2589
+ if (last && unit.startLine <= last.endLine) {
2590
+ last.units.push(unit);
2591
+ if (unit.endLine > last.endLine) last.endLine = unit.endLine;
2592
+ } else {
2593
+ groups.push({ units: [unit], startLine: unit.startLine, endLine: unit.endLine, records: null });
2594
+ }
2595
+ }
2596
+ for (let i = 0; i < groups.length; i++) for (const unit of groups[i].units) unit.group = i;
2597
+ return groups;
2598
+ }
2599
+
2600
+ /**
2601
+ * Read groups of an isolated context. Groups whose expanded text parses
2602
+ * without an error read together as alone (each unit is an item of its
2603
+ * container); otherwise each group is read alone.
2604
+ */
2605
+ function readIsolatedGroups(ctx, targets) {
2606
+ if (targets.length === 1) {
2607
+ readIsolatedGroup(ctx, targets[0]);
2608
+ return;
2609
+ }
2610
+ const rec = { blind: [], used: new Map() };
2611
+ const wanted = new Set(targets);
2612
+ const mine = new Set(targets.flatMap(i => ctx.groups[i].slots));
2613
+ const expand = slot => expandSlot(ctx, slot, rec);
2614
+ const live = ctx.slots.filter(slot => mine.has(slot)).map(expand);
2615
+ const keep = analysisUnits(live, 'calls').keep;
2616
+ const extras = ctx.slots.filter(slot => !mine.has(slot) &&
2617
+ keep.some(([start, end]) => slot.start < end && slot.end > start)).map(expand);
2618
+ const expanded = live.concat(extras);
2619
+ const bySlot = new Map(expanded.map(region => [region.slot, region]));
2620
+ const all = ctx.slots.map(slot => bySlot.get(slot) || placeholderOf(ctx, slot));
2621
+ const built = buildSynthetic(ctx.content, all, analysisUnits(expanded, 'calls').keep);
2622
+ const synthTree = parseSynthetic(ctx.parser, built.text, built.spans);
2623
+ if (collectErrorRanges(synthTree.rootNode, []).length > 0) {
2624
+ for (const i of targets) readIsolatedGroup(ctx, i);
2625
+ return;
2626
+ }
2627
+ const units = [];
2628
+ for (const i of targets) {
2629
+ const group = ctx.groups[i];
2630
+ const regions = live.filter(region => region.slot && ctx.groups[i].slots.includes(region.slot));
2631
+ group.units = analysisUnits(regions, 'calls').units;
2632
+ for (const unit of group.units) unit.group = i;
2633
+ group.starts = regions.map(region => region.start);
2634
+ group.records = [];
2635
+ units.push(...group.units);
2636
+ }
2637
+ units.sort((a, b) => a.start - b.start);
2638
+ const lookups = unitLookups(units);
2639
+ const originalSpanless = new Map();
2640
+ for (const record of ctx.original) {
2641
+ if (Number.isFinite(record.callStart) || !lookups.unitAtLine(record.line)) continue;
2642
+ const key = spanlessKey(record);
2643
+ originalSpanless.set(key, (originalSpanless.get(key) || 0) + 1);
2644
+ }
2645
+ const rust = require('../languages/rust');
2646
+ const synthCalls = rust.findCallsInCode(built.text, ctx.parser, {
2647
+ tree: synthTree, itemRecovery: rust.rustExpandedItemRecovery(built.text, ctx.parser, synthTree),
2648
+ });
2649
+ mergeRecords(ctx, synthCalls, {
2650
+ mapper: offsetMapper(all), lookups, regions: live, synthTree, originalSpanless,
2651
+ accept: unit => wanted.has(unit.group), push: (unit, call) => ctx.groups[unit.group].records.push(call),
2652
+ });
2653
+ }
2654
+
2655
+ /** Expand one invocation of an isolated context as the build did. */
2656
+ function expandSlot(ctx, slot, rec) {
2657
+ ctx.state.hygieneSeq = slot.item.hygiene;
2658
+ const region = expandPlanned(ctx.state, ctx.filePath, ctx.content, slot.item, rec);
2659
+ if (!region || region.layout.text.length !== slot.item.length) {
2660
+ throw new Error(`Rust macro expansion at ${ctx.filePath}:${slot.startLine} differs from the build's`);
2661
+ }
2662
+ region.slot = slot;
2663
+ return region;
2664
+ }
2665
+
2666
+ /** A blank of an expansion's length standing in for it (its newlines kept). */
2667
+ function placeholderOf(ctx, slot) {
2668
+ if (!ctx.placeholders) ctx.placeholders = new Map();
2669
+ let region = ctx.placeholders.get(slot);
2670
+ if (!region) {
2671
+ const breaks = slot.endLine - slot.startLine;
2672
+ region = {
2673
+ start: slot.start, end: slot.end, startLine: slot.startLine, endLine: slot.endLine, blank: true,
2674
+ layout: { text: ' '.repeat(slot.item.length - breaks) + '\n'.repeat(breaks), marks: [] },
2675
+ };
2676
+ ctx.placeholders.set(slot, region);
2677
+ }
2678
+ // Its offsets are set by the synthetic it stands in.
2679
+ return { ...region };
2680
+ }
2681
+
2682
+ /** Read one group of an isolated context: its expansions, parsed alone. */
2683
+ function readIsolatedGroup(ctx, gi) {
2684
+ const group = ctx.groups[gi];
2685
+ const rec = { blind: [], used: new Map() };
2686
+ const expand = slot => expandSlot(ctx, slot, rec);
2687
+ const mine = new Set(group.slots);
2688
+ let live = group.slots.map(expand);
2689
+ // Expansions inside the kept headers of this group's units
2690
+ // (`impl T for m!() { .. }`) are part of its text.
2691
+ const headerKeep = analysisUnits(live, 'calls').keep;
2692
+ let extras = ctx.slots.filter(slot => !mine.has(slot) &&
2693
+ headerKeep.some(([start, end]) => slot.start < end && slot.end > start)).map(expand);
2694
+ let synthetic = null;
2695
+ let synthTree = null;
2696
+ let all = null;
2697
+ for (let attempt = 0; live.length > 0 && attempt <= ctx.slots.length; attempt++) {
2698
+ const expanded = live.concat(extras);
2699
+ const bySlot = new Map(expanded.map(region => [region.slot, region]));
2700
+ const dropped = new Set(group.slots.filter(slot => !bySlot.has(slot)));
2701
+ // A dropped expansion stays as written, as in one pass.
2702
+ all = ctx.slots.filter(slot => !dropped.has(slot)).map(slot => bySlot.get(slot) || placeholderOf(ctx, slot));
2703
+ const built = buildSynthetic(ctx.content, all, analysisUnits(expanded, 'calls').keep);
2704
+ synthetic = built.text;
2705
+ synthTree = parseSynthetic(ctx.parser, synthetic, built.spans);
2706
+ const errors = collectErrorRanges(synthTree.rootNode, []);
2707
+ const bad = new Set();
2708
+ for (const [start, end] of errors) {
2709
+ for (const region of expanded) {
2710
+ if (start < region.synthEnd && end > region.synthStart) bad.add(region);
2711
+ else if (start === end && start >= region.synthStart && start <= region.synthEnd) bad.add(region);
2712
+ }
2713
+ }
2714
+ if (bad.size === 0) break;
2715
+ live = live.filter(region => !bad.has(region));
2716
+ extras = extras.filter(region => !bad.has(region));
2717
+ }
2718
+ group.units = live.length > 0 ? analysisUnits(live, 'calls').units : [];
2719
+ group.starts = live.map(region => region.start);
2720
+ group.records = [];
2721
+ if (live.length === 0) return;
2722
+ const lookups = unitLookups(group.units);
2723
+ const mapper = offsetMapper(all);
2724
+ const originalSpanless = new Map();
2725
+ for (const record of ctx.original) {
2726
+ if (Number.isFinite(record.callStart) || !lookups.unitAtLine(record.line)) continue;
2727
+ const key = spanlessKey(record);
2728
+ originalSpanless.set(key, (originalSpanless.get(key) || 0) + 1);
2729
+ }
2730
+ const rust = require('../languages/rust');
2731
+ const synthCalls = rust.findCallsInCode(synthetic, ctx.parser, {
2732
+ tree: synthTree, itemRecovery: rust.rustExpandedItemRecovery(synthetic, ctx.parser, synthTree),
2733
+ });
2734
+ mergeRecords(ctx, synthCalls, {
2735
+ mapper, lookups, regions: live, synthTree, originalSpanless,
2736
+ accept: () => true, push: (unit, call) => group.records.push(call),
2737
+ });
2738
+ }
2739
+
2740
+ /**
2741
+ * Words the records of an isolated group can bear: its lines and its units'
2742
+ * kept headers in the file (literals and comments left out), the text of
2743
+ * every project macro its expansions can transcribe, and the crate names
2744
+ * `$crate` becomes.
2745
+ */
2746
+ function isolatedGroupWords(ctx, group) {
2747
+ if (group.words) return group.words;
2748
+ const state = ctx.state;
2749
+ const words = new Set();
2750
+ const ranges = analysisUnits(group.slots, 'calls').keep.concat([[
2751
+ lineOffset(state, ctx.filePath, group.startLine), lineEndOffset(state, ctx.filePath, group.endLine),
2752
+ ]]).sort((a, b) => a[0] - b[0]);
2753
+ if (!ctx.nonCode) ctx.nonCode = ctx.tree.rootNode.descendantsOfType(NON_CODE_TOKENS);
2754
+ const nonCode = ctx.nonCode;
2755
+ const scan = (start, end) => {
2756
+ if (end <= start) return;
2757
+ for (const word of ctx.content.slice(start, end).match(WORD) || []) words.add(word);
2758
+ };
2759
+ let next = 0;
2760
+ let done = 0;
2761
+ for (const [rangeStart, end] of ranges) {
2762
+ let at = Math.max(rangeStart, done);
2763
+ if (end <= at) continue;
2764
+ while (next < nonCode.length && nonCode[next].endIndex <= at) next++;
2765
+ for (let i = next; i < nonCode.length; i++) {
2766
+ const token = nonCode[i];
2767
+ if (token.startIndex >= end) break;
2768
+ scan(at, Math.min(token.startIndex, end));
2769
+ at = Math.max(at, token.endIndex);
2770
+ }
2771
+ scan(at, end);
2772
+ done = end;
2773
+ }
2774
+ for (const word of [...words]) if (state.inventory.has(word)) addTemplateWords(state, word, words);
2775
+ words.add('crate');
2776
+ group.words = words;
2777
+ return words;
2778
+ }
2779
+
2780
+ /** Words of every definition of macro `name` and the macros they name, transitively. */
2781
+ function addTemplateWords(state, name, words) {
2782
+ if (!state.templateWords) state.templateWords = new Map();
2783
+ let own = state.templateWords.get(name);
2784
+ if (!own) {
2785
+ own = new Set();
2786
+ state.templateWords.set(name, own);
2787
+ const pending = [name];
2788
+ const visited = new Set();
2789
+ while (pending.length > 0) {
2790
+ const next = pending.pop();
2791
+ if (visited.has(next)) continue;
2792
+ visited.add(next);
2793
+ for (const def of state.inventory.get(next) || []) {
2794
+ const content = readContent(state, def.file);
2795
+ if (content === null) continue;
2796
+ const text = content.slice(lineOffset(state, def.file, def.startLine), lineOffset(state, def.file, def.endLine + 1));
2797
+ for (const word of text.match(WORD) || []) {
2798
+ own.add(word);
2799
+ if (state.inventory.has(word) && !visited.has(word)) pending.push(word);
2800
+ }
2801
+ const crate = packageOf(state, def.file)?.packageName;
2802
+ if (crate) own.add(crate);
2803
+ }
2804
+ }
2805
+ }
2806
+ for (const word of own) words.add(word);
2807
+ }
2808
+
2809
+ /**
2810
+ * The records of a file whose expansion calls are pending that a consumer
2811
+ * reading only part of them needs (fix #405), in the order the composed
2812
+ * calls hold them and as the same record objects. `name`: the records
2813
+ * `keep` accepts, each of which must bear the name as one of its own
2814
+ * identifier tokens (its name or written receiver); `lines`: [start, end],
2815
+ * the records on those lines; `lineSet`: the records on any of those lines. Only the groups that can hold such records
2816
+ * are read. undefined when the file's calls are not pending, `name` is not
2817
+ * an identifier, or every group is read (the file's calls are then derived
2818
+ * whole by the caller).
2819
+ */
2820
+ const IDENTIFIER = /^(?:r#)?[\p{L}_][\p{L}\p{N}_]*$/u;
2821
+ // Tenths of a file's unread expanded text from which a view derives the file
2822
+ // whole (fix #405b).
2823
+ const WHOLE_FILE_SHARE = 9;
2824
+
2825
+ function rustPendingRecords(index, filePath, { name = null, keep = null, lines = null, lineSet = null } = {}) {
2826
+ const entry = index.files.get(filePath);
2827
+ if (!entry?.rustMacroExpansion?.callsPending) return undefined;
2828
+ // Hygiene renames locals a transcriber introduces (`x__ucn_h3`): no
2829
+ // source text holds such a name.
2830
+ if (name !== null && (!IDENTIFIER.test(name) || /__ucn_h\d+$/.test(name))) return undefined;
2831
+ const raw = index.callsCache.get(filePath)?.calls;
2832
+ if (!Array.isArray(raw)) return undefined;
2833
+ const matches = record => (!lines || (record.line >= lines[0] && record.line <= lines[1])) &&
2834
+ (!lineSet || lineSet.has(record.line)) && (!keep || keep(record));
2835
+ // No derived record bears a name outside the file's words (#375): the
2836
+ // parsed records then answer, unless one of them bears it.
2837
+ if (name !== null && !rustPendingMayBear(entry, name) && !raw.some(matches)) return [];
2838
+ // The groups recorded at build (fix #405b) are chosen before anything is
2839
+ // parsed: a group can hold a record `matches` accepts when a word its
2840
+ // expansions output or its headers hold is the name, or when a parsed
2841
+ // record on its lines is accepted (the units' own code is written there).
2842
+ const rec = entry.rustMacroExpansion;
2843
+ const recorded = Array.isArray(rec.groups) ? rec.groups : null;
2844
+ let planned = null;
2845
+ if (recorded) {
2846
+ const read = index._rustMacroState?.contexts?.get(filePath)?.groups || null;
2847
+ const parsedOn = name !== null ? parsedRecordLines(raw, matches) : null;
2848
+ planned = [];
2849
+ let wantedLength = 0;
2850
+ let unreadLength = 0;
2851
+ for (let i = 0; i < recorded.length; i++) {
2852
+ const [startLine, endLine, length] = recorded[i];
2853
+ const unread = !read?.[i]?.records;
2854
+ if (unread) unreadLength += length;
2855
+ if (lines && (endLine < lines[0] || startLine > lines[1])) continue;
2856
+ if (lineSet && ![...lineSet].some(line => line >= startLine && line <= endLine)) continue;
2857
+ if (name !== null && !groupHasWord(recorded[i][3], name) &&
2858
+ !parsedOn.some(line => line >= startLine && line <= endLine)) continue;
2859
+ planned.push(i);
2860
+ if (unread) wantedLength += length;
2861
+ }
2862
+ if (planned.length === 0) {
2863
+ // Every group read: the caller derives the file's calls whole.
2864
+ if (read && read.length > 0 && read.every(group => group.records)) return undefined;
2865
+ // No group can hold one: the accepted records are parsed ones
2866
+ // outside every unit, as the composed calls hold them.
2867
+ return raw.filter(matches).sort((a, b) => a.line - b.line);
2868
+ }
2869
+ // Reading most of the expanded text that is still unread costs more
2870
+ // than deriving the file whole (groups read together cost ~5-10%
2871
+ // over one pass), which then answers every later query of the file:
2872
+ // a cost decision, not a semantic one - either way the records are
2873
+ // the composed calls'.
2874
+ if (wantedLength > 0 && wantedLength * 10 >= unreadLength * WHOLE_FILE_SHARE) return undefined;
2875
+ }
2876
+ const ctx = pendingContext(index, filePath);
2877
+ if (!ctx) return raw.filter(matches);
2878
+ // Every group read: the caller derives the file's calls whole.
2879
+ if (ctx.groups.every(group => group.records)) return undefined;
2880
+ let wanted;
2881
+ if (planned && ctx.groups.length === recorded.length && ctx.groups.every((group, i) =>
2882
+ group.startLine === recorded[i][0] && group.endLine === recorded[i][1])) {
2883
+ wanted = planned;
2884
+ } else {
2885
+ wanted = [];
2886
+ for (let i = 0; i < ctx.groups.length; i++) {
2887
+ const group = ctx.groups[i];
2888
+ if (lines && (group.endLine < lines[0] || group.startLine > lines[1])) continue;
2889
+ if (lineSet && ![...lineSet].some(line => line >= group.startLine && line <= group.endLine)) continue;
2890
+ if (name !== null && !groupMayBear(ctx, group, name)) continue;
2891
+ wanted.push(i);
2892
+ }
2893
+ }
2894
+ readGroups(ctx, wanted);
2895
+ if (ctx.groups.every(group => group.records)) return undefined;
2896
+ const out = ctx.original.filter(record => matches(record) && !inReadUnit(ctx, record));
2897
+ for (const i of wanted) for (const record of ctx.groups[i].records) if (matches(record)) out.push(record);
2898
+ for (const record of invocationRecords(ctx)) if (matches(record)) out.push(record);
2899
+ return out.sort((a, b) => a.line - b.line);
2900
+ }
2901
+
2902
+ /**
2903
+ * Whether a group's records can bear `name`: a word of the group's lines in
2904
+ * the expanded file or of the kept text outside every unit (headers: a
2905
+ * `Self` record bears its impl's type name).
2906
+ */
2907
+ function groupMayBear(ctx, group, name) {
2908
+ if (ctx.isolated) return hasWord(isolatedGroupWords(ctx, group), name);
2909
+ if (!ctx.outsideWords) {
2910
+ const words = new Set();
2911
+ let at = 0;
2912
+ for (const unit of ctx.units) {
2913
+ for (const word of ctx.synthetic.slice(at, unit.synthStart).match(WORD) || []) words.add(word);
2914
+ at = Math.max(at, unit.synthEnd);
2915
+ }
2916
+ for (const word of ctx.synthetic.slice(at).match(WORD) || []) words.add(word);
2917
+ ctx.outsideWords = words;
2918
+ }
2919
+ if (hasWord(ctx.outsideWords, name)) return true;
2920
+ if (!group.words) {
2921
+ if (!ctx.synthLineStarts) {
2922
+ const starts = [0];
2923
+ const text = ctx.synthetic;
2924
+ for (let i = text.indexOf('\n'); i >= 0; i = text.indexOf('\n', i + 1)) starts.push(i + 1);
2925
+ ctx.synthLineStarts = starts;
2926
+ }
2927
+ const starts = ctx.synthLineStarts;
2928
+ const from = starts[Math.max(group.startLine - 1, 0)] ?? ctx.synthetic.length;
2929
+ const to = starts[group.endLine] ?? ctx.synthetic.length;
2930
+ group.words = new Set(ctx.synthetic.slice(from, to).match(WORD) || []);
2931
+ }
2932
+ return hasWord(group.words, name);
2933
+ }
2934
+
2935
+ /** Lines of the parsed records `matches` accepts. */
2936
+ function parsedRecordLines(raw, matches) {
2937
+ const out = [];
2938
+ for (const record of raw) if (matches(record)) out.push(record.line);
2939
+ return out;
2940
+ }
2941
+
2942
+ function hasWord(words, name) {
2943
+ // A raw identifier's token is `r#name`; text holds both parts.
2944
+ return words.has(name) || words.has(String(name).replace(/^r#/, '')) || words.has(`r#${name}`);
2945
+ }
2946
+
2947
+ /**
2948
+ * Relative paths of the files whose expansion calls are pending with no
2949
+ * part derived yet, sorted.
2950
+ */
2951
+ function rustUntouchedPendingFiles(index) {
2952
+ const contexts = index._rustMacroState?.contexts;
2953
+ const out = [];
2954
+ for (const [filePath, entry] of index.files) {
2955
+ if (!entry.rustMacroExpansion?.callsPending) continue;
2956
+ if (contexts?.get(filePath)?.groups?.some(group => group.records)) continue;
2957
+ out.push(entry.relativePath);
2958
+ }
2959
+ return out.sort(codeUnitCompare);
2960
+ }
2961
+
2962
+ /** Derive whole every file whose expansion calls were derived in part. */
2963
+ function completeRustMacroCalls(index) {
2964
+ const contexts = index._rustMacroState?.contexts;
2965
+ if (!contexts || contexts.size === 0) return 0;
2966
+ let done = 0;
2967
+ for (const filePath of [...contexts.keys()].sort(codeUnitCompare)) {
2968
+ if (!index.files.get(filePath)?.rustMacroExpansion?.callsPending) continue;
2969
+ const cached = index.callsCache.get(filePath);
2970
+ if (!cached) continue;
2971
+ rustExpandedCalls(index, filePath, cached.calls);
2972
+ done++;
2973
+ }
2974
+ return done;
2975
+ }
2976
+
2977
+ /** Derive every pending file's expansion calls (whole-program consumers). */
2978
+ function materializeRustMacroCalls(index) {
2979
+ for (const [filePath, entry] of index.files) {
2980
+ if (!entry.rustMacroExpansion?.callsPending) continue;
2981
+ const cached = index.callsCache.get(filePath);
2982
+ if (cached) rustExpandedCalls(index, filePath, cached.calls);
2983
+ }
2984
+ }
2985
+
2986
+ /** Expanded and blind invocation sites, for health and deadcode disclosure. */
2987
+ function rustMacroExpansionSummary(index) {
2988
+ const summary = { sites: 0, files: 0, blind: { count: 0, fileCount: 0, files: [], sample: [], reasons: {} } };
2989
+ const files = [...index.files.values()].filter(entry => entry.rustMacroExpansion)
2990
+ .sort((a, b) => codeUnitCompare(a.relativePath, b.relativePath));
2991
+ for (const entry of files) {
2992
+ const rec = entry.rustMacroExpansion;
2993
+ summary.sites += rec.sites || 0;
2994
+ if (rec.expanded) summary.files++;
2995
+ const blind = rec.blind || [];
2996
+ if (blind.length === 0) continue;
2997
+ summary.blind.count += blind.length;
2998
+ summary.blind.fileCount++;
2999
+ if (summary.blind.files.length < 10) summary.blind.files.push(entry.relativePath);
3000
+ for (const site of blind) {
3001
+ const reason = site.reason.startsWith('nested:') ? 'nested' : site.reason;
3002
+ summary.blind.reasons[reason] = (summary.blind.reasons[reason] || 0) + 1;
3003
+ if (summary.blind.sample.length < BLIND_SAMPLE) {
3004
+ summary.blind.sample.push({ file: entry.relativePath, ...site });
3005
+ }
3006
+ }
3007
+ }
3008
+ return summary;
3009
+ }
3010
+
3011
+ module.exports = {
3012
+ EXPANSION_WORKERS,
3013
+ planRustMacroExpansion,
3014
+ startRustMacroExpansion,
3015
+ // worker-side (core/build-worker.js)
3016
+ runExpansionQueue,
3017
+ applyRustMacroExpansion,
3018
+ rustExpandedCalls,
3019
+ materializeRustMacroCalls,
3020
+ rustMacroExpansionSummary,
3021
+ rustPendingMayBear,
3022
+ rustPendingFilesBearing,
3023
+ rustPendingRecords,
3024
+ completeRustMacroCalls,
3025
+ rustUntouchedPendingFiles,
3026
+ };