ucn 5.4.1 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/ucn/SKILL.md +58 -236
- package/.claude/skills/ucn/references/commands.md +144 -68
- package/.claude/skills/ucn/references/resolution-rules.md +171 -0
- package/.claude/skills/ucn/references/trust-contract.md +27 -33
- package/README.md +18 -12
- package/cli/index.js +42 -16
- package/core/accessors.js +118 -3
- package/core/account.js +30 -0
- package/core/analysis.js +508 -264
- package/core/ast-analysis.js +76 -117
- package/core/async-producers.js +1729 -0
- package/core/bridge.js +1877 -268
- package/core/brief.js +6 -0
- package/core/build-worker.js +78 -14
- package/core/cache.js +522 -31
- package/core/callers.js +12053 -1047
- package/core/check.js +30 -38
- package/core/class-identity.js +1028 -0
- package/core/command-contracts.js +6 -6
- package/core/configuration-items.js +86 -0
- package/core/contract-membership.js +2262 -0
- package/core/cpp-scope.js +532 -0
- package/core/deadcode.js +171 -49
- package/core/decorator-binding.js +260 -0
- package/core/discovery.js +281 -164
- package/core/errors.js +52 -0
- package/core/execute.js +21 -9
- package/core/external-macros.js +326 -0
- package/core/graph-build.js +277 -8
- package/core/graph.js +35 -2
- package/core/ignore-rules.js +472 -0
- package/core/imports.js +504 -42
- package/core/index-ir.js +94 -11
- package/core/inprocess-clients.js +636 -0
- package/core/ir.js +87 -8
- package/core/macro-expansion.js +1416 -0
- package/core/output/analysis-ext.js +6 -6
- package/core/output/analysis.js +18 -2
- package/core/output/brief.js +3 -0
- package/core/output/doctor.js +4 -0
- package/core/output/endpoints.js +15 -6
- package/core/output/find.js +11 -2
- package/core/output/lines.js +8 -2
- package/core/output/refactoring.js +121 -14
- package/core/output/reporting.js +31 -3
- package/core/output/shared.js +14 -0
- package/core/output-budget.js +316 -122
- package/core/parallel-build.js +124 -47
- package/core/parser.js +3 -2
- package/core/project.js +457 -60
- package/core/provenance-facts.js +156 -12
- package/core/provenance-overload.js +30 -3
- package/core/provenance.js +104 -30
- package/core/pytest-fixtures.js +434 -0
- package/core/python-fixture-flow.js +17 -3
- package/core/python-modules.js +173 -0
- package/core/reflection.js +877 -0
- package/core/registry.js +1 -1
- package/core/reporting.js +234 -52
- package/core/request-helpers.js +561 -0
- package/core/route-graph.js +1922 -0
- package/core/route-lists.js +2414 -0
- package/core/rust-macro-expansion.js +2113 -0
- package/core/rust-modules.js +185 -0
- package/core/search.js +121 -29
- package/core/shared.js +19 -1
- package/core/test-name-bindings.js +256 -0
- package/core/type-denotation.js +1161 -0
- package/core/type-references.js +3229 -0
- package/core/verify.js +3360 -672
- package/languages/adapter.js +31 -5
- package/languages/c-family.js +4061 -212
- package/languages/c-macro-invocations.js +380 -0
- package/languages/c-preprocessor.js +254 -0
- package/languages/csharp.js +1406 -49
- package/languages/go.js +1243 -70
- package/languages/index.js +728 -6
- package/languages/java.js +473 -61
- package/languages/javascript.js +1270 -197
- package/languages/lexical-scope.js +1280 -0
- package/languages/python.js +1557 -72
- package/languages/rust-macro-rules.js +1020 -0
- package/languages/rust-value-flow.js +416 -0
- package/languages/rust.js +2237 -115
- package/languages/utils.js +298 -4
- package/mcp/server.js +25 -11
- package/package.json +1 -1
|
@@ -0,0 +1,2113 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Project-local Rust `macro_rules!` expansion (fix #374).
|
|
5
|
+
*
|
|
6
|
+
* A project macro invocation generates code the AST never shows: impl blocks
|
|
7
|
+
* (`delegate_iterator! { Iter<'a, K, V> => .., impl<..> }`), test functions
|
|
8
|
+
* (`quickcheck! { fn prop(a: Iter<u16>) -> bool { .. } }`), forwarding
|
|
9
|
+
* methods (`deref_forward_buf!();` inside an impl) and calls whose receivers
|
|
10
|
+
* only get a type once the arguments sit in their expanded context. This
|
|
11
|
+
* module expands such invocations at build time and makes what they generate
|
|
12
|
+
* ordinary index facts:
|
|
13
|
+
*
|
|
14
|
+
* 1. Invocations are the grammar's `macro_invocation` nodes whose macro
|
|
15
|
+
* resolves to a project `macro_rules!` by Rust's own scoping: textual
|
|
16
|
+
* scope (earlier in the file, `#[macro_use] mod` children, the parent
|
|
17
|
+
* module up to the `mod` declaration), then path scope
|
|
18
|
+
* (`#[macro_export]` macros through `crate::`/`$crate::`/package paths,
|
|
19
|
+
* `use` bindings, globs, `#[macro_use] extern crate`).
|
|
20
|
+
* 2. languages/rust-macro-rules.js matches the invocation's token trees
|
|
21
|
+
* against the rules and transcribes; project macros invoked by the
|
|
22
|
+
* transcription expand recursively (bounded depth and output), locals
|
|
23
|
+
* the transcriber introduces are renamed apart (hygiene) and `$crate`
|
|
24
|
+
* names the defining crate.
|
|
25
|
+
* 3. The file is re-analyzed with every expansion spliced in place of its
|
|
26
|
+
* invocation, laid out on the invocation's own lines (argument tokens
|
|
27
|
+
* keep their line), so the full Rust parser extracts functions, impls,
|
|
28
|
+
* calls and receiver facts in their real context. The file's symbols
|
|
29
|
+
* and call records become those of the expanded file: facts produced
|
|
30
|
+
* inside an invocation carry `macroExpansion {macro, definition, origin,
|
|
31
|
+
* line}` (origin 'argument' when the token was written in the
|
|
32
|
+
* invocation, 'template' when it came from the transcriber).
|
|
33
|
+
*
|
|
34
|
+
* An invocation whose rules do not match, whose expansion exceeds the bounds
|
|
35
|
+
* or does not parse in its context stays as written (the parser's token-tree
|
|
36
|
+
* view) and is recorded as blind, disclosed by `repo --sections=health
|
|
37
|
+
* --deep` and deadcode. Macros from dependencies (`vec!`, `println!`, a
|
|
38
|
+
* crate's `quickcheck!`) are never expanded: their definitions are not in the
|
|
39
|
+
* project. Results are persisted with the file entry and revalidated against
|
|
40
|
+
* the file, every macro definition it used and its module ancestry.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
const crypto = require('crypto');
|
|
44
|
+
const fs = require('fs');
|
|
45
|
+
const path = require('path');
|
|
46
|
+
const { getParser, safeParse, PARSE_OPTIONS } = require('../languages');
|
|
47
|
+
const { addIRSymbol } = require('./index-ir');
|
|
48
|
+
const { findCargoRoot } = require('./imports');
|
|
49
|
+
const {
|
|
50
|
+
MacroBlind, argTokensOf, compileMacro, expandWithRules, isPostfixChain, tokenCount,
|
|
51
|
+
} = require('../languages/rust-macro-rules');
|
|
52
|
+
const { codeUnitCompare } = require('./shared');
|
|
53
|
+
|
|
54
|
+
const MAX_DEPTH = 64;
|
|
55
|
+
const MAX_OUTPUT_TOKENS = 20000;
|
|
56
|
+
const BLIND_SAMPLE = 20;
|
|
57
|
+
|
|
58
|
+
function isRust(fileEntry) {
|
|
59
|
+
return fileEntry?.language === 'rust';
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** name -> Rust macro_rules! definitions. */
|
|
63
|
+
function macroInventory(index) {
|
|
64
|
+
const inventory = new Map();
|
|
65
|
+
for (const [name, defs] of index.symbols) {
|
|
66
|
+
for (const def of defs) {
|
|
67
|
+
if (def.type !== 'macro') continue;
|
|
68
|
+
if (!isRust(index.files.get(def.file))) continue;
|
|
69
|
+
if (!inventory.has(name)) inventory.set(name, []);
|
|
70
|
+
inventory.get(name).push(def);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return inventory;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function defKey(def) {
|
|
77
|
+
return `${def.relativePath}:${def.startLine}`;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ── Per-pass state ────────────────────────────────────────────────────────
|
|
81
|
+
|
|
82
|
+
function createState(index, inventory = null) {
|
|
83
|
+
return {
|
|
84
|
+
index,
|
|
85
|
+
parser: getParser('rust'),
|
|
86
|
+
inventory: inventory || macroInventory(index),
|
|
87
|
+
contents: new Map(),
|
|
88
|
+
trees: new Map(),
|
|
89
|
+
compiled: new Map(),
|
|
90
|
+
parents: null,
|
|
91
|
+
packages: new Map(), // file dir -> cargo info
|
|
92
|
+
crateNames: null, // crate name -> package root dir
|
|
93
|
+
roots: new Map(), // file -> crate root file
|
|
94
|
+
macroUseCrates: new Map(), // crate root file -> Set<crate name>
|
|
95
|
+
lineStarts: new Map(),
|
|
96
|
+
textualMemo: new Map(),
|
|
97
|
+
productive: new Map(),
|
|
98
|
+
hygieneSeq: 0,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function readContent(state, file) {
|
|
103
|
+
let content = state.contents.get(file);
|
|
104
|
+
if (content === undefined) {
|
|
105
|
+
try {
|
|
106
|
+
content = state.index._readFile(file);
|
|
107
|
+
} catch {
|
|
108
|
+
content = null;
|
|
109
|
+
}
|
|
110
|
+
state.contents.set(file, content);
|
|
111
|
+
}
|
|
112
|
+
return content;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function treeOf(state, file) {
|
|
116
|
+
if (state.trees.has(file)) return state.trees.get(file);
|
|
117
|
+
const content = readContent(state, file);
|
|
118
|
+
const tree = content === null ? null : safeParse(state.parser, content, undefined, PARSE_OPTIONS);
|
|
119
|
+
state.trees.set(file, tree);
|
|
120
|
+
return tree;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** child module file -> { file, line, macroUse } from `mod x;` declarations. */
|
|
124
|
+
function moduleParents(state) {
|
|
125
|
+
if (state.parents) return state.parents;
|
|
126
|
+
const parents = new Map();
|
|
127
|
+
const index = state.index;
|
|
128
|
+
for (const [file, entry] of index.files) {
|
|
129
|
+
if (!isRust(entry) || !entry.moduleResolved) continue;
|
|
130
|
+
for (const detail of entry.importDetails || []) {
|
|
131
|
+
if (detail.type !== 'mod') continue;
|
|
132
|
+
const rel = entry.moduleResolved[detail.module];
|
|
133
|
+
if (!rel) continue;
|
|
134
|
+
const child = path.join(index.root, rel);
|
|
135
|
+
if (child === file || parents.has(child)) continue;
|
|
136
|
+
const decl = (entry.symbols || []).find(symbol => symbol.type === 'module' &&
|
|
137
|
+
symbol.name === detail.module && symbol.startLine === detail.line);
|
|
138
|
+
parents.set(child, {
|
|
139
|
+
file, line: detail.line,
|
|
140
|
+
macroUse: !!decl?.modifiers?.includes('macro_use'),
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
state.parents = parents;
|
|
145
|
+
return parents;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function crateRoot(state, file) {
|
|
149
|
+
if (state.roots.has(file)) return state.roots.get(file);
|
|
150
|
+
const parents = moduleParents(state);
|
|
151
|
+
let current = file;
|
|
152
|
+
const seen = new Set();
|
|
153
|
+
while (parents.has(current) && !seen.has(current)) {
|
|
154
|
+
seen.add(current);
|
|
155
|
+
current = parents.get(current).file;
|
|
156
|
+
}
|
|
157
|
+
state.roots.set(file, current);
|
|
158
|
+
return current;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function packageOf(state, file) {
|
|
162
|
+
const dir = path.dirname(file);
|
|
163
|
+
if (!state.packages.has(dir)) state.packages.set(dir, findCargoRoot(dir) || null);
|
|
164
|
+
return state.packages.get(dir);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function packageByCrateName(state, name) {
|
|
168
|
+
if (!state.crateNames) {
|
|
169
|
+
state.crateNames = new Map();
|
|
170
|
+
for (const [file, entry] of state.index.files) {
|
|
171
|
+
if (!isRust(entry)) continue;
|
|
172
|
+
const info = packageOf(state, file);
|
|
173
|
+
if (info?.packageName && !state.crateNames.has(info.packageName)) {
|
|
174
|
+
state.crateNames.set(info.packageName, info.root);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return state.crateNames.get(name) || null;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function isLibRoot(state, rootFile) {
|
|
182
|
+
const info = packageOf(state, rootFile);
|
|
183
|
+
return !!info && rootFile === path.join(info.srcDir, 'lib.rs');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Tokens `$crate` becomes for a definition expanded at `siteFile`. */
|
|
187
|
+
function crateTokensFor(state, def, siteFile) {
|
|
188
|
+
const defRoot = crateRoot(state, def.file);
|
|
189
|
+
if (defRoot === crateRoot(state, siteFile) || !isLibRoot(state, defRoot)) {
|
|
190
|
+
return [{ k: 'ident', v: 'crate' }];
|
|
191
|
+
}
|
|
192
|
+
const name = packageOf(state, def.file)?.packageName;
|
|
193
|
+
return [{ k: 'ident', v: name || 'crate' }];
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// ── Definitions ───────────────────────────────────────────────────────────
|
|
197
|
+
|
|
198
|
+
function attributeNames(node) {
|
|
199
|
+
const names = [];
|
|
200
|
+
for (let sibling = node.previousNamedSibling; sibling; sibling = sibling.previousNamedSibling) {
|
|
201
|
+
if (sibling.type === 'line_comment' || sibling.type === 'block_comment') continue;
|
|
202
|
+
if (sibling.type !== 'attribute_item') break;
|
|
203
|
+
const attribute = sibling.namedChildren.find(child => child.type === 'attribute');
|
|
204
|
+
const head = attribute?.namedChildren.find(child =>
|
|
205
|
+
child.type === 'identifier' || child.type === 'scoped_identifier');
|
|
206
|
+
if (head) names.push(head.text);
|
|
207
|
+
}
|
|
208
|
+
return names;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Compiled definition + lexical scope, memoized per pass. */
|
|
212
|
+
function compiledDef(state, def) {
|
|
213
|
+
const key = defKey(def);
|
|
214
|
+
if (state.compiled.has(key)) return state.compiled.get(key);
|
|
215
|
+
// Attributes and textual scope are parser facts of the definition; the
|
|
216
|
+
// rules are compiled from the definition's own text, parsed alone with
|
|
217
|
+
// its lines kept (no re-parse of the defining file).
|
|
218
|
+
const attrs = def.modifiers || [];
|
|
219
|
+
const start = lineOffset(state, def.file, def.startLine);
|
|
220
|
+
const end = lineOffset(state, def.file, def.endLine + 1);
|
|
221
|
+
const scope = def.macroScope ? {
|
|
222
|
+
kind: def.macroScope.kind,
|
|
223
|
+
start: lineOffset(state, def.file, def.macroScope.startLine),
|
|
224
|
+
end: lineOffset(state, def.file, def.macroScope.endLine + 1),
|
|
225
|
+
...(def.macroScope.macroUse && { macroUse: true }),
|
|
226
|
+
} : null;
|
|
227
|
+
const base = { start, end, exported: attrs.includes('macro_export'), cfg: attrs.includes('cfg'), scope };
|
|
228
|
+
let result;
|
|
229
|
+
const content = readContent(state, def.file);
|
|
230
|
+
const text = content === null ? null
|
|
231
|
+
: '\n'.repeat(def.startLine - 1) + content.slice(start, end);
|
|
232
|
+
const tree = text === null ? null : safeParse(state.parser, text, undefined, PARSE_OPTIONS);
|
|
233
|
+
const node = tree && tree.rootNode.descendantsOfType('macro_definition').find(candidate =>
|
|
234
|
+
candidate.startPosition.row + 1 === def.startLine &&
|
|
235
|
+
candidate.childForFieldName('name')?.text === def.name);
|
|
236
|
+
if (!node) {
|
|
237
|
+
result = { ...base, error: 'definition-unavailable' };
|
|
238
|
+
} else {
|
|
239
|
+
try {
|
|
240
|
+
result = { ...base, compiled: compileMacro(node) };
|
|
241
|
+
} catch (error) {
|
|
242
|
+
if (!(error instanceof MacroBlind)) throw error;
|
|
243
|
+
result = { ...base, error: error.reason };
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
state.compiled.set(key, result);
|
|
247
|
+
return result;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ── Resolution ────────────────────────────────────────────────────────────
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Definitions of `name` in textual scope at (file, offset); `offset`
|
|
254
|
+
* Infinity asks what a module exports to its parent through #[macro_use].
|
|
255
|
+
* Returns [{def, pos}] candidates of the innermost scope that has any.
|
|
256
|
+
*/
|
|
257
|
+
function textualCandidates(state, file, offset, line, name, walkParents, seen = new Set()) {
|
|
258
|
+
if (seen.has(file)) return [];
|
|
259
|
+
seen.add(file);
|
|
260
|
+
const candidates = [];
|
|
261
|
+
for (const def of state.inventory.get(name) || []) {
|
|
262
|
+
if (def.file !== file) continue;
|
|
263
|
+
const info = compiledDef(state, def);
|
|
264
|
+
if (info.start === undefined) continue;
|
|
265
|
+
if (offset !== Infinity && info.end > offset) continue;
|
|
266
|
+
const scope = info.scope;
|
|
267
|
+
if (scope) {
|
|
268
|
+
const inside = offset >= scope.start && offset <= scope.end;
|
|
269
|
+
const after = scope.kind === 'mod' && scope.macroUse && offset >= scope.end;
|
|
270
|
+
if (offset === Infinity ? !(scope.kind === 'mod' && scope.macroUse) : !(inside || after)) continue;
|
|
271
|
+
}
|
|
272
|
+
candidates.push({ def, pos: info.start, info });
|
|
273
|
+
}
|
|
274
|
+
// `#[macro_use] mod child;` before this point brings the child's macros.
|
|
275
|
+
const entry = state.index.files.get(file);
|
|
276
|
+
const parents = moduleParents(state);
|
|
277
|
+
for (const detail of entry?.importDetails || []) {
|
|
278
|
+
if (detail.type !== 'mod' || !(detail.line < line || line === Infinity)) continue;
|
|
279
|
+
const rel = entry.moduleResolved?.[detail.module];
|
|
280
|
+
if (!rel) continue;
|
|
281
|
+
const child = path.join(state.index.root, rel);
|
|
282
|
+
const link = parents.get(child);
|
|
283
|
+
if (!link || link.file !== file || !link.macroUse) continue;
|
|
284
|
+
const exported = textualCandidates(state, child, Infinity, Infinity, name, false, seen);
|
|
285
|
+
// Position: the declaration line (same-line declarations are not
|
|
286
|
+
// idiomatic), keeping the child's own order among its definitions
|
|
287
|
+
// so its cfg alternatives stay together.
|
|
288
|
+
const declared = lineOffset(state, file, detail.line);
|
|
289
|
+
for (const candidate of exported) {
|
|
290
|
+
candidates.push({ ...candidate, pos: declared + candidate.pos / 1e12 });
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (candidates.length > 0) return candidates;
|
|
294
|
+
if (!walkParents) return [];
|
|
295
|
+
const parent = parents.get(file);
|
|
296
|
+
if (!parent) return [];
|
|
297
|
+
// What the parent sees at the child's `mod` declaration does not depend
|
|
298
|
+
// on where in the child the invocation is.
|
|
299
|
+
const key = `${parent.file}\0${parent.line}\0${name}`;
|
|
300
|
+
if (!state.textualMemo.has(key)) {
|
|
301
|
+
state.textualMemo.set(key, textualCandidates(state, parent.file,
|
|
302
|
+
lineOffset(state, parent.file, parent.line), parent.line, name, true, new Set(seen)));
|
|
303
|
+
}
|
|
304
|
+
return state.textualMemo.get(key);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function lineOffset(state, file, line) {
|
|
308
|
+
let starts = state.lineStarts.get(file);
|
|
309
|
+
if (!starts) {
|
|
310
|
+
const content = readContent(state, file);
|
|
311
|
+
starts = [0];
|
|
312
|
+
if (content !== null) {
|
|
313
|
+
for (let i = content.indexOf('\n'); i >= 0; i = content.indexOf('\n', i + 1)) starts.push(i + 1);
|
|
314
|
+
}
|
|
315
|
+
state.lineStarts.set(file, starts);
|
|
316
|
+
}
|
|
317
|
+
return starts[Math.min(Math.max(line - 1, 0), starts.length - 1)];
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function pickLatest(candidates) {
|
|
321
|
+
return candidates.reduce((best, candidate) => (!best || candidate.pos > best.pos ? candidate : best), null);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Definitions a bare or path invocation of `name` resolves to at
|
|
326
|
+
* (file, offset). Returns { defs, ambiguous } or null (not a project macro
|
|
327
|
+
* in scope).
|
|
328
|
+
*/
|
|
329
|
+
function resolveMacro(state, file, offset, line, name, qualifier, dollarCratePackage) {
|
|
330
|
+
const all = state.inventory.get(name);
|
|
331
|
+
if (!all || all.length === 0) return null;
|
|
332
|
+
const inPackage = pkgRoot => all.filter(def => packageOf(state, def.file)?.root === pkgRoot);
|
|
333
|
+
const preferExported = defs => {
|
|
334
|
+
const exported = defs.filter(def => compiledDef(state, def).exported);
|
|
335
|
+
return exported.length > 0 ? exported : defs;
|
|
336
|
+
};
|
|
337
|
+
const finish = defs => {
|
|
338
|
+
if (defs.length === 0) return null;
|
|
339
|
+
return { defs: defs.slice(0, 4), ambiguous: defs.length > 1 };
|
|
340
|
+
};
|
|
341
|
+
if (qualifier) {
|
|
342
|
+
const first = qualifier.split('::')[0];
|
|
343
|
+
const pkgRoot = dollarCratePackage ||
|
|
344
|
+
(['crate', 'self', 'super', '$crate'].includes(first) ? packageOf(state, file)?.root
|
|
345
|
+
: packageByCrateName(state, first));
|
|
346
|
+
if (!pkgRoot) return null;
|
|
347
|
+
return finish(preferExported(inPackage(pkgRoot)));
|
|
348
|
+
}
|
|
349
|
+
const textual = textualCandidates(state, file, offset, line, name, true);
|
|
350
|
+
if (textual.length > 0) {
|
|
351
|
+
const best = pickLatest(textual);
|
|
352
|
+
// Configuration alternatives: several cfg-gated definitions visible
|
|
353
|
+
// from the same scope.
|
|
354
|
+
const alternatives = best.info?.cfg
|
|
355
|
+
? textual.filter(candidate => candidate.info?.cfg && candidate.def.file === best.def.file)
|
|
356
|
+
: [best];
|
|
357
|
+
// Every configuration's definition expands: cfg-gated code is code.
|
|
358
|
+
return {
|
|
359
|
+
defs: alternatives.map(candidate => candidate.def).slice(0, 4),
|
|
360
|
+
alternatives: alternatives.length > 1,
|
|
361
|
+
ambiguous: false,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
const own = packageOf(state, file);
|
|
365
|
+
const root = crateRoot(state, file);
|
|
366
|
+
const entry = state.index.files.get(file);
|
|
367
|
+
// #[macro_export] macros live in their crate root's namespace.
|
|
368
|
+
if (own && root === file) {
|
|
369
|
+
const exported = inPackage(own.root).filter(def => compiledDef(state, def).exported &&
|
|
370
|
+
crateRoot(state, def.file) === root);
|
|
371
|
+
if (exported.length > 0) return finish(exported);
|
|
372
|
+
}
|
|
373
|
+
// `use crate::m;` / `use pkg::m;` / `use pkg::path::m as n;`
|
|
374
|
+
for (const binding of entry?.importBindings || []) {
|
|
375
|
+
const local = binding.alias || binding.name;
|
|
376
|
+
if (local !== name) continue;
|
|
377
|
+
const first = String(binding.module || '').split('::')[0];
|
|
378
|
+
const pkgRoot = ['crate', 'self', 'super'].includes(first) ? own?.root
|
|
379
|
+
: packageByCrateName(state, first);
|
|
380
|
+
if (!pkgRoot) continue;
|
|
381
|
+
const found = finish(preferExported(inPackage(pkgRoot)));
|
|
382
|
+
if (found) return found;
|
|
383
|
+
}
|
|
384
|
+
// `use pkg::*;` and `#[macro_use] extern crate pkg;`
|
|
385
|
+
const crates = new Set();
|
|
386
|
+
for (const detail of entry?.importDetails || []) {
|
|
387
|
+
if (detail.type === 'use-glob' && !String(detail.module).includes('::')) crates.add(detail.module);
|
|
388
|
+
}
|
|
389
|
+
for (const crate of macroUseCrates(state, root)) crates.add(crate);
|
|
390
|
+
for (const crate of crates) {
|
|
391
|
+
const pkgRoot = crate === 'crate' ? own?.root : packageByCrateName(state, crate);
|
|
392
|
+
if (!pkgRoot) continue;
|
|
393
|
+
const exported = inPackage(pkgRoot).filter(def => compiledDef(state, def).exported);
|
|
394
|
+
const found = finish(exported);
|
|
395
|
+
if (found) return found;
|
|
396
|
+
}
|
|
397
|
+
return null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** Crates whose macros a crate root imports with `#[macro_use] extern crate`. */
|
|
401
|
+
function macroUseCrates(state, rootFile) {
|
|
402
|
+
if (state.macroUseCrates.has(rootFile)) return state.macroUseCrates.get(rootFile);
|
|
403
|
+
const crates = new Set();
|
|
404
|
+
const content = readContent(state, rootFile);
|
|
405
|
+
if (content !== null && content.includes('macro_use')) {
|
|
406
|
+
const tree = treeOf(state, rootFile);
|
|
407
|
+
for (const node of tree?.rootNode.namedChildren || []) {
|
|
408
|
+
if (node.type !== 'extern_crate_declaration') continue;
|
|
409
|
+
if (!attributeNames(node).includes('macro_use')) continue;
|
|
410
|
+
const name = node.childForFieldName('name')?.text;
|
|
411
|
+
if (name) crates.add(name);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
state.macroUseCrates.set(rootFile, crates);
|
|
415
|
+
return crates;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// ── Productivity ──────────────────────────────────────────────────────────
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Whether expanding a macro can change the facts of the code it expands to
|
|
422
|
+
* beyond the parser's token-tree view of the invocation (whose argument
|
|
423
|
+
* calls are already recorded and typed in scope, and whose template calls
|
|
424
|
+
* are already visible at the definition): it declares items, binds argument
|
|
425
|
+
* fragments as locals or parameters, places an argument in callee position
|
|
426
|
+
* (`$f(..)`, `x.$m(..)`), or invokes a project macro that does.
|
|
427
|
+
* Other macros (logging, `try!`-style matches, assertions) are not expanded.
|
|
428
|
+
*/
|
|
429
|
+
/**
|
|
430
|
+
* true (always worth expanding), 'callee' (only where an argument names a
|
|
431
|
+
* project callable) or false.
|
|
432
|
+
*/
|
|
433
|
+
function productivity(state, def, visiting = new Set()) {
|
|
434
|
+
const key = defKey(def);
|
|
435
|
+
if (state.productive.has(key)) return state.productive.get(key);
|
|
436
|
+
if (visiting.has(key)) return false;
|
|
437
|
+
visiting.add(key);
|
|
438
|
+
const info = compiledDef(state, def);
|
|
439
|
+
let productive;
|
|
440
|
+
if (info.compiled) {
|
|
441
|
+
productive = info.compiled.declares || info.compiled.bindsArgs ? true
|
|
442
|
+
: info.compiled.callsArgs ? 'callee' : false;
|
|
443
|
+
for (const name of info.compiled.invokes) {
|
|
444
|
+
if (productive === true) break;
|
|
445
|
+
for (const other of state.inventory.get(name) || []) {
|
|
446
|
+
const nested = productivity(state, other, visiting);
|
|
447
|
+
if (nested === true) productive = true;
|
|
448
|
+
else if (nested === 'callee' && !productive) productive = 'callee';
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
} else {
|
|
452
|
+
// A definition the engine cannot model is expanded (and disclosed).
|
|
453
|
+
productive = true;
|
|
454
|
+
}
|
|
455
|
+
state.productive.set(key, productive);
|
|
456
|
+
return productive;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Whether an invocation of `defs` is worth expanding given its source text:
|
|
461
|
+
* a macro that only moves arguments into callee position is expanded where
|
|
462
|
+
* an argument word names a project callable (a word scan of the invocation
|
|
463
|
+
* is only a prefilter; the expansion decides what is a call).
|
|
464
|
+
*/
|
|
465
|
+
function worthExpanding(state, defs, text, expressionContext) {
|
|
466
|
+
let calleeOnly = false;
|
|
467
|
+
for (const def of defs) {
|
|
468
|
+
const level = productivity(state, def);
|
|
469
|
+
if (level === true) return true;
|
|
470
|
+
if (level === 'callee') calleeOnly = true;
|
|
471
|
+
}
|
|
472
|
+
if (expressionContext && forwardsValue(state, defs)) return true;
|
|
473
|
+
if (!calleeOnly || !text) return false;
|
|
474
|
+
for (const word of new Set(text.match(/[A-Za-z_][A-Za-z0-9_]*/g) || [])) {
|
|
475
|
+
if (namesCallable(state.index, word)) return true;
|
|
476
|
+
}
|
|
477
|
+
return false;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
const NON_CALLABLE_DEF_TYPES = new Set(['macro', 'struct', 'enum', 'trait', 'module', 'type', 'impl', 'field']);
|
|
481
|
+
|
|
482
|
+
/** Whether a project definition named `word` can be called. */
|
|
483
|
+
function namesCallable(index, word) {
|
|
484
|
+
if (index.callableNames) return index.callableNames.has(word);
|
|
485
|
+
const defs = index.symbols.get(word);
|
|
486
|
+
return !!defs && defs.some(def => !NON_CALLABLE_DEF_TYPES.has(def.type));
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/** An expression-position invocation whose value is one of its arguments. */
|
|
490
|
+
function forwardsValue(state, defs) {
|
|
491
|
+
return defs.some(def => compiledDef(state, def).compiled?.forwardsValue);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Whether any recorded invocation of the file can need expanding, decided
|
|
496
|
+
* from call records before the file is parsed again.
|
|
497
|
+
*/
|
|
498
|
+
function fileMayExpand(state, filePath) {
|
|
499
|
+
const calls = state.index.callsCache.get(filePath)?.calls || [];
|
|
500
|
+
let memberRanges = null;
|
|
501
|
+
const inImplOrTrait = line => {
|
|
502
|
+
memberRanges ||= (state.index.files.get(filePath)?.symbols || [])
|
|
503
|
+
.filter(symbol => symbol.type === 'impl' || symbol.type === 'trait')
|
|
504
|
+
.map(symbol => [symbol.startLine, symbol.endLine]);
|
|
505
|
+
return memberRanges.some(([start, end]) => line >= start && line <= end);
|
|
506
|
+
};
|
|
507
|
+
for (const call of calls) {
|
|
508
|
+
if (!call.isMacro || call.inMacro || call.macroExpansion || !state.inventory.has(call.name)) continue;
|
|
509
|
+
if (!Number.isInteger(call.callStart)) return true;
|
|
510
|
+
const qualifier = call.isPathMacro ? call.receiver : null;
|
|
511
|
+
const resolved = resolveMacro(state, filePath, call.callStart, call.line, call.name, qualifier, null);
|
|
512
|
+
if (!resolved) continue;
|
|
513
|
+
// Item wrappers (`cfg_x! { items }`) at module level declare what the
|
|
514
|
+
// parser already recovers from their token trees.
|
|
515
|
+
if (resolved.defs.every(def => compiledDef(state, def).compiled?.passThroughItems) &&
|
|
516
|
+
!call.enclosingFunction && !inImplOrTrait(call.line)) continue;
|
|
517
|
+
const content = readContent(state, filePath);
|
|
518
|
+
const text = content === null ? null : content.slice(call.callStart, call.callEnd || call.callStart);
|
|
519
|
+
// Expression position is a parser fact of the invocation record.
|
|
520
|
+
if (worthExpanding(state, resolved.defs, text, call.macroExpr === true)) return true;
|
|
521
|
+
}
|
|
522
|
+
return false;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// ── Expansion ─────────────────────────────────────────────────────────────
|
|
526
|
+
|
|
527
|
+
function invocationName(node) {
|
|
528
|
+
const macroNode = node.childForFieldName('macro');
|
|
529
|
+
if (!macroNode) return null;
|
|
530
|
+
const parts = macroNode.text.split('::').filter(Boolean);
|
|
531
|
+
const name = parts.pop();
|
|
532
|
+
if (!name) return null;
|
|
533
|
+
return { name, qualifier: parts.length > 0 ? parts.join('::') : null };
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/** Syntactic context the invocation's expansion is parsed in. */
|
|
537
|
+
function invocationContext(node) {
|
|
538
|
+
return require('../languages/rust').rustMacroInvocationContext(node);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function itemRecoveryPosition(node) {
|
|
542
|
+
if (node.parent?.type === 'source_file') return true;
|
|
543
|
+
return node.parent?.type === 'declaration_list' && node.parent.parent?.type === 'mod_item';
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function isTok(tok, v) {
|
|
547
|
+
return tok && tok.k !== 'group' && tok.v === v;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* Expand `toks` (an invocation's token list) with one definition, then every
|
|
552
|
+
* project macro the transcription invokes. `site` carries the budgets, the
|
|
553
|
+
* used definitions and blind records of this top-level invocation.
|
|
554
|
+
*/
|
|
555
|
+
function expandTokens(state, site, def, toks, depth) {
|
|
556
|
+
if (depth > MAX_DEPTH) throw new MacroBlind('recursion-depth');
|
|
557
|
+
const info = compiledDef(state, def);
|
|
558
|
+
if (info.error) throw new MacroBlind(info.error);
|
|
559
|
+
site.used.set(defKey(def), def);
|
|
560
|
+
const ctx = {
|
|
561
|
+
macro: def.name,
|
|
562
|
+
hygiene: ++state.hygieneSeq,
|
|
563
|
+
crateTokens: crateTokensFor(state, def, site.file),
|
|
564
|
+
budget: site.budget,
|
|
565
|
+
};
|
|
566
|
+
const { tokens } = expandWithRules(info.compiled, toks, ctx);
|
|
567
|
+
site.budget -= tokenCount(tokens);
|
|
568
|
+
if (site.budget < 0) throw new MacroBlind('output-budget');
|
|
569
|
+
return expandNested(state, site, def, tokens, depth);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function expandNested(state, site, def, tokens, depth) {
|
|
573
|
+
const out = [];
|
|
574
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
575
|
+
const tok = tokens[i];
|
|
576
|
+
if (tok.k === 'group') {
|
|
577
|
+
out.push({ ...tok, c: expandNested(state, site, def, tok.c, depth) });
|
|
578
|
+
continue;
|
|
579
|
+
}
|
|
580
|
+
if (tok.k !== 'ident' || isTok(tokens[i - 1], '::') || tok.v === 'macro_rules') {
|
|
581
|
+
out.push(tok);
|
|
582
|
+
continue;
|
|
583
|
+
}
|
|
584
|
+
let k = i;
|
|
585
|
+
const segments = [tok.v];
|
|
586
|
+
while (isTok(tokens[k + 1], '::') && tokens[k + 2]?.k === 'ident') {
|
|
587
|
+
k += 2;
|
|
588
|
+
segments.push(tokens[k].v);
|
|
589
|
+
}
|
|
590
|
+
const group = tokens[k + 2];
|
|
591
|
+
if (!isTok(tokens[k + 1], '!') || group?.k !== 'group') {
|
|
592
|
+
out.push(tok);
|
|
593
|
+
continue;
|
|
594
|
+
}
|
|
595
|
+
const name = segments.pop();
|
|
596
|
+
const qualifier = segments.length > 0 ? segments.join('::') : null;
|
|
597
|
+
const dollarCratePackage = tok.dollarCrate ? packageOf(state, def.file)?.root : null;
|
|
598
|
+
let resolved = resolveMacro(state, site.file, site.offset, site.line, name, qualifier, dollarCratePackage);
|
|
599
|
+
if (!resolved && !qualifier) {
|
|
600
|
+
const info = compiledDef(state, def);
|
|
601
|
+
resolved = resolveMacro(state, def.file, info.start ?? 0, def.startLine, name, null, null);
|
|
602
|
+
}
|
|
603
|
+
if (!resolved) {
|
|
604
|
+
out.push(tok);
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
607
|
+
const prev = out[out.length - 1];
|
|
608
|
+
const following = tokens[k + 3];
|
|
609
|
+
// Statement/item position: at the start of a token list or after a
|
|
610
|
+
// statement, as a braced invocation or one ending its statement.
|
|
611
|
+
const statementPosition = (!prev || isTok(prev, ';') || (prev.k === 'group' && prev.d === '{')) &&
|
|
612
|
+
(group.d === '{' || !following || isTok(following, ';'));
|
|
613
|
+
let inner;
|
|
614
|
+
try {
|
|
615
|
+
inner = expandAlternatives(state, site, resolved, group.c, depth + 1,
|
|
616
|
+
statementPosition ? 'stmts' : 'expr');
|
|
617
|
+
if (resolved.ambiguous) site.ambiguous = true;
|
|
618
|
+
} catch (error) {
|
|
619
|
+
if (!(error instanceof MacroBlind)) throw error;
|
|
620
|
+
if (error.reason === 'output-budget') throw error;
|
|
621
|
+
site.nestedBlind.push({ macro: name, reason: error.reason });
|
|
622
|
+
out.push(tok);
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
out.push(...(statementPosition ? inner : wrapExpression(inner)));
|
|
626
|
+
i = k + 2;
|
|
627
|
+
}
|
|
628
|
+
return out;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Expand with every configuration alternative of a macro. Items and
|
|
633
|
+
* statements of each alternative are all kept (each is real code under its
|
|
634
|
+
* configuration); differing expression values become the arms of one
|
|
635
|
+
* `match`, so no configuration's calls are lost. Throws MacroBlind when no
|
|
636
|
+
* alternative expands.
|
|
637
|
+
*/
|
|
638
|
+
function expandAlternatives(state, site, resolved, toks, depth, context) {
|
|
639
|
+
const outputs = [];
|
|
640
|
+
const texts = new Set();
|
|
641
|
+
let failure = null;
|
|
642
|
+
for (const def of resolved.defs) {
|
|
643
|
+
try {
|
|
644
|
+
const tokens = expandTokens(state, site, def, cloneTree(toks), depth);
|
|
645
|
+
if (resolved.defs.length > 1) {
|
|
646
|
+
const text = tokenText(tokens);
|
|
647
|
+
if (texts.has(text)) continue;
|
|
648
|
+
texts.add(text);
|
|
649
|
+
}
|
|
650
|
+
outputs.push(tokens);
|
|
651
|
+
} catch (error) {
|
|
652
|
+
if (!(error instanceof MacroBlind) || error.reason === 'output-budget') throw error;
|
|
653
|
+
failure = failure || error;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
if (outputs.length === 0) throw failure || new MacroBlind('no-matching-rule');
|
|
657
|
+
if (failure) site.nestedBlind.push({ macro: resolved.defs[0].name, reason: `alternative:${failure.reason}` });
|
|
658
|
+
if (outputs.length === 1) return outputs[0];
|
|
659
|
+
if (context !== 'expr') return outputs.flat();
|
|
660
|
+
const arms = [];
|
|
661
|
+
outputs.forEach((tokens, i) => {
|
|
662
|
+
if (i > 0) arms.push({ k: 'punct', v: ',', src: 'tpl' });
|
|
663
|
+
arms.push(i === outputs.length - 1 ? { k: 'ident', v: '_', src: 'tpl' } : { k: 'lit', v: String(i), src: 'tpl' },
|
|
664
|
+
{ k: 'punct', v: '=>', src: 'tpl' }, ...wrapExpression(tokens));
|
|
665
|
+
});
|
|
666
|
+
return [{ k: 'group', v: '(', d: '(', src: 'tpl', synthetic: true, c: [
|
|
667
|
+
{ k: 'ident', v: 'match', src: 'tpl' }, { k: 'lit', v: '0', src: 'tpl' },
|
|
668
|
+
{ k: 'group', v: '{', d: '{', src: 'tpl', synthetic: true, c: arms },
|
|
669
|
+
] }];
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function cloneTree(tokens) {
|
|
673
|
+
return tokens.map(tok => (tok.k === 'group' ? { ...tok, c: cloneTree(tok.c) } : { ...tok }));
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
/**
|
|
677
|
+
* An expansion used as an expression: a block holding a single expression
|
|
678
|
+
* (`{{ Command::new() }}`) is that expression; anything that is not one
|
|
679
|
+
* postfix operand keeps its precedence in parentheses, statements in a block.
|
|
680
|
+
*/
|
|
681
|
+
function wrapExpression(input) {
|
|
682
|
+
let tokens = input;
|
|
683
|
+
while (tokens.length === 1 && tokens[0].k === 'group' && tokens[0].d === '{' &&
|
|
684
|
+
tokens[0].c.length > 0 && !tokens[0].c.some(tok => isTok(tok, ';') || isTok(tok, 'let'))) {
|
|
685
|
+
tokens = tokens[0].c;
|
|
686
|
+
}
|
|
687
|
+
if (tokens.length === 1 && tokens[0].k === 'group') return tokens;
|
|
688
|
+
if (isPostfixChain(tokens)) return tokens;
|
|
689
|
+
const hasStatement = tokens.some(tok => isTok(tok, ';'));
|
|
690
|
+
const d = hasStatement ? '{' : '(';
|
|
691
|
+
return [{ k: 'group', v: d, d, c: tokens, src: 'tpl', synthetic: true }];
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// Hygiene: locals a transcriber introduces (`let x`, closure parameters,
|
|
695
|
+
// `for x in`) live in that expansion step's syntax context and never bind
|
|
696
|
+
// the invocation's tokens of the same spelling (or vice versa).
|
|
697
|
+
const RUST_KEYWORDS = new Set([
|
|
698
|
+
'as', 'async', 'await', 'break', 'const', 'continue', 'crate', 'dyn', 'else', 'enum', 'extern',
|
|
699
|
+
'false', 'fn', 'for', 'if', 'impl', 'in', 'let', 'loop', 'match', 'mod', 'move', 'mut', 'pub',
|
|
700
|
+
'ref', 'return', 'self', 'Self', 'static', 'struct', 'super', 'trait', 'true', 'type', 'unsafe',
|
|
701
|
+
'use', 'where', 'while', '_',
|
|
702
|
+
]);
|
|
703
|
+
|
|
704
|
+
function renameHygienicLocals(tokens) {
|
|
705
|
+
const binders = new Set();
|
|
706
|
+
const collect = list => {
|
|
707
|
+
for (let i = 0; i < list.length; i++) {
|
|
708
|
+
const tok = list[i];
|
|
709
|
+
if (tok.k === 'group') {
|
|
710
|
+
collect(tok.c);
|
|
711
|
+
if (isTok(list[i - 1], 'let') || (isTok(list[i - 1], 'mut') && isTok(list[i - 2], 'let'))) {
|
|
712
|
+
for (const inner of tok.c) {
|
|
713
|
+
if (inner.k === 'ident' && inner.src === 'tpl' && !RUST_KEYWORDS.has(inner.v) &&
|
|
714
|
+
!/^[A-Z]/.test(inner.v)) {
|
|
715
|
+
binders.add(`${inner.ctx}\0${inner.v}`);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
if (tok.src !== 'tpl' || tok.k !== 'ident' || RUST_KEYWORDS.has(tok.v)) continue;
|
|
722
|
+
const prev = list[i - 1];
|
|
723
|
+
const prev2 = list[i - 2];
|
|
724
|
+
const next = list[i + 1];
|
|
725
|
+
// `let Some(x)` / `let Point { x, .. }` / `let a::B`: a path,
|
|
726
|
+
// not a binding (the bindings sit in the pattern's group).
|
|
727
|
+
const pathHead = (next?.k === 'group' && next.d !== '[') || isTok(next, '::');
|
|
728
|
+
if (pathHead && (isTok(prev, 'let') || (isTok(prev, 'mut') && isTok(prev2, 'let')))) {
|
|
729
|
+
for (const inner of next.c || []) {
|
|
730
|
+
if (inner.k === 'ident' && inner.src === 'tpl' && !RUST_KEYWORDS.has(inner.v) &&
|
|
731
|
+
!/^[A-Z]/.test(inner.v)) binders.add(`${inner.ctx}\0${inner.v}`);
|
|
732
|
+
}
|
|
733
|
+
continue;
|
|
734
|
+
}
|
|
735
|
+
if (isTok(prev, 'let') || (isTok(prev, 'mut') && isTok(prev2, 'let')) ||
|
|
736
|
+
(isTok(prev, 'for') && isTok(next, 'in'))) {
|
|
737
|
+
binders.add(`${tok.ctx}\0${tok.v}`);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
// closure parameters: `|a, mut b: T|`
|
|
741
|
+
for (let i = 0; i < list.length; i++) {
|
|
742
|
+
if (!isTok(list[i], '|') || list[i].src !== 'tpl') continue;
|
|
743
|
+
const prev = list[i - 1];
|
|
744
|
+
if (prev && prev.k !== 'group' && !['(', ',', '=', 'move', '=>', ';'].includes(prev.v) &&
|
|
745
|
+
prev.k !== 'punct') continue;
|
|
746
|
+
let j = i + 1;
|
|
747
|
+
let expectName = true;
|
|
748
|
+
while (j < list.length && !isTok(list[j], '|')) {
|
|
749
|
+
const tok = list[j];
|
|
750
|
+
if (isTok(tok, ',')) expectName = true;
|
|
751
|
+
else if (expectName && tok.k === 'ident' && !RUST_KEYWORDS.has(tok.v)) {
|
|
752
|
+
if (tok.src === 'tpl') binders.add(`${tok.ctx}\0${tok.v}`);
|
|
753
|
+
expectName = false;
|
|
754
|
+
} else if (!isTok(tok, 'mut') && !isTok(tok, 'ref') && !isTok(tok, '&')) {
|
|
755
|
+
expectName = false;
|
|
756
|
+
}
|
|
757
|
+
j++;
|
|
758
|
+
}
|
|
759
|
+
i = j;
|
|
760
|
+
}
|
|
761
|
+
};
|
|
762
|
+
collect(tokens);
|
|
763
|
+
if (binders.size === 0) return tokens;
|
|
764
|
+
const rename = list => {
|
|
765
|
+
for (let i = 0; i < list.length; i++) {
|
|
766
|
+
const tok = list[i];
|
|
767
|
+
if (tok.k === 'group') {
|
|
768
|
+
rename(tok.c);
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
if (tok.k !== 'ident' || tok.src !== 'tpl' || RUST_KEYWORDS.has(tok.v) ||
|
|
772
|
+
!binders.has(`${tok.ctx}\0${tok.v}`)) continue;
|
|
773
|
+
if (isTok(list[i - 1], '.') || isTok(list[i - 1], '::') ||
|
|
774
|
+
isTok(list[i + 1], '::') || isTok(list[i + 1], '!')) continue;
|
|
775
|
+
tok.v = `${tok.v}__ucn_h${tok.ctx}`;
|
|
776
|
+
}
|
|
777
|
+
};
|
|
778
|
+
rename(tokens);
|
|
779
|
+
return tokens;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// ── Layout ────────────────────────────────────────────────────────────────
|
|
783
|
+
|
|
784
|
+
const CLOSE = { '(': ')', '[': ']', '{': '}' };
|
|
785
|
+
|
|
786
|
+
function flatten(tokens, out = []) {
|
|
787
|
+
for (const tok of tokens) {
|
|
788
|
+
if (tok.k === 'group') {
|
|
789
|
+
out.push({ v: tok.d, src: tok.src, line: tok.line, col: tok.col, s: tok.s, e: tok.s !== undefined ? tok.s + 1 : undefined });
|
|
790
|
+
flatten(tok.c, out);
|
|
791
|
+
const close = tok.closeOrigin;
|
|
792
|
+
out.push(close && tok.src === 'arg'
|
|
793
|
+
? { v: CLOSE[tok.d], src: 'arg', line: close.line, col: close.col, s: close.s, e: close.e, closer: true }
|
|
794
|
+
: { v: CLOSE[tok.d], src: 'tpl', closer: true });
|
|
795
|
+
} else {
|
|
796
|
+
out.push(tok);
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
return out;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
const MULTI_PUNCT = ['::', '->', '=>', '==', '!=', '<=', '>=', '&&', '||', '+=', '-=', '*=', '/=',
|
|
803
|
+
'%=', '^=', '&=', '|=', '<<', '>>', '..', '//', '/*', '*/'];
|
|
804
|
+
const SPACE_AFTER = new Set([',', ';', ':', '=>', '->', '=', '+', '|', '&&', '||', '==', '!=']);
|
|
805
|
+
const SPACE_BEFORE = new Set(['=>', '->', '=', '+', '{', '==', '!=', '&&', '||']);
|
|
806
|
+
// ASCII letters, digits, '_', quotes (and '#' at a word's end): a leaf that
|
|
807
|
+
// ends / starts with one lexes into its neighbor without a space.
|
|
808
|
+
function isWordCode(code) {
|
|
809
|
+
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90) || (code >= 48 && code <= 57) ||
|
|
810
|
+
code === 95 || code === 39 || code === 34;
|
|
811
|
+
}
|
|
812
|
+
const isWordEnd = leaf => {
|
|
813
|
+
const v = typeof leaf.v === 'string' ? leaf.v : String(leaf.v);
|
|
814
|
+
if (v.length === 0) return false;
|
|
815
|
+
const code = v.charCodeAt(v.length - 1);
|
|
816
|
+
return isWordCode(code) || code === 35;
|
|
817
|
+
};
|
|
818
|
+
const isWordStart = leaf => {
|
|
819
|
+
const v = typeof leaf.v === 'string' ? leaf.v : String(leaf.v);
|
|
820
|
+
return v.length > 0 && isWordCode(v.charCodeAt(0));
|
|
821
|
+
};
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* Whether two adjacent leaves need a space: when they would lex as one
|
|
825
|
+
* token, plus conventional spacing so text-derived facts (parameter lists,
|
|
826
|
+
* generics, paths) read like source.
|
|
827
|
+
*/
|
|
828
|
+
function needsSpace(prev, leaf) {
|
|
829
|
+
if (!prev) return false;
|
|
830
|
+
const wordEnd = isWordEnd(prev);
|
|
831
|
+
const wordStart = isWordStart(leaf);
|
|
832
|
+
if (wordEnd && wordStart) return true;
|
|
833
|
+
const a = prev.v;
|
|
834
|
+
const b = leaf.v;
|
|
835
|
+
if (b === '.' && /^[0-9]/.test(a)) return true;
|
|
836
|
+
if (!wordEnd && !wordStart) {
|
|
837
|
+
const joint = a.slice(-1) + b[0];
|
|
838
|
+
if (MULTI_PUNCT.some(p => p.startsWith(joint)) && !(a === '::' || b === '::')) return true;
|
|
839
|
+
}
|
|
840
|
+
if (a === ':' && b === ':') return true;
|
|
841
|
+
if (SPACE_AFTER.has(a) && a !== '::') return !(a === '|' && prev.closer);
|
|
842
|
+
if ((a === '>' || a === '}') && wordStart) return true;
|
|
843
|
+
if (SPACE_BEFORE.has(b)) return true;
|
|
844
|
+
return false;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/**
|
|
848
|
+
* Text of an expansion laid out on the invocation's lines [startLine,
|
|
849
|
+
* endLine]: argument tokens move to their own line when it is ahead, leading
|
|
850
|
+
* template tokens join the next argument token's line, nothing ever passes
|
|
851
|
+
* endLine. Returns { text, marks: [{s, e, leaf}] } (offsets relative).
|
|
852
|
+
*/
|
|
853
|
+
function layout(tokens, startLine, startCol, endLine) {
|
|
854
|
+
const leaves = flatten(tokens);
|
|
855
|
+
const nextArgLine = new Array(leaves.length + 1).fill(0);
|
|
856
|
+
for (let i = leaves.length - 1; i >= 0; i--) {
|
|
857
|
+
nextArgLine[i] = leaves[i].src === 'arg' ? leaves[i].line : nextArgLine[i + 1];
|
|
858
|
+
}
|
|
859
|
+
let line = startLine;
|
|
860
|
+
let col = startCol;
|
|
861
|
+
let text = '';
|
|
862
|
+
const marks = [];
|
|
863
|
+
let first = true;
|
|
864
|
+
for (let i = 0; i < leaves.length; i++) {
|
|
865
|
+
const leaf = leaves[i];
|
|
866
|
+
let target = 0;
|
|
867
|
+
if (leaf.src === 'arg' && leaf.line > line) target = leaf.line;
|
|
868
|
+
else if (leaf.src !== 'arg' && !leaf.closer && nextArgLine[i + 1] > line) target = nextArgLine[i + 1];
|
|
869
|
+
if (target > line && target <= endLine) {
|
|
870
|
+
text += '\n'.repeat(target - line);
|
|
871
|
+
line = target;
|
|
872
|
+
col = 0;
|
|
873
|
+
if (leaf.src === 'arg' && leaf.col > col) {
|
|
874
|
+
text += ' '.repeat(leaf.col);
|
|
875
|
+
col = leaf.col;
|
|
876
|
+
}
|
|
877
|
+
} else if (!first && needsSpace(leaves[i - 1], leaf)) {
|
|
878
|
+
text += ' ';
|
|
879
|
+
col++;
|
|
880
|
+
}
|
|
881
|
+
first = false;
|
|
882
|
+
let value = leaf.v;
|
|
883
|
+
if (value.includes('\n')) {
|
|
884
|
+
const breaks = value.split('\n').length - 1;
|
|
885
|
+
if (leaf.src === 'arg' && line + breaks <= endLine) {
|
|
886
|
+
line += breaks;
|
|
887
|
+
col = value.length - value.lastIndexOf('\n') - 1;
|
|
888
|
+
} else {
|
|
889
|
+
value = /^b?r?#*"|^b?"/.test(value) ? '""' : ' ';
|
|
890
|
+
col += value.length;
|
|
891
|
+
}
|
|
892
|
+
} else {
|
|
893
|
+
col += value.length;
|
|
894
|
+
}
|
|
895
|
+
marks.push({ s: text.length, e: text.length + value.length, leaf, line });
|
|
896
|
+
text += value;
|
|
897
|
+
}
|
|
898
|
+
if (line < endLine) text += '\n'.repeat(endLine - line);
|
|
899
|
+
return { text, marks };
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// ── Per-file expansion ────────────────────────────────────────────────────
|
|
903
|
+
|
|
904
|
+
function collectErrorRanges(node, out) {
|
|
905
|
+
if (node.type === 'ERROR' || node.isMissing) {
|
|
906
|
+
out.push([node.startIndex, node.endIndex]);
|
|
907
|
+
return out;
|
|
908
|
+
}
|
|
909
|
+
if (!node.hasError) return out;
|
|
910
|
+
for (let i = 0; i < node.childCount; i++) collectErrorRanges(node.child(i), out);
|
|
911
|
+
return out;
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/**
|
|
915
|
+
* The expanded file as analyzed: only the kept ranges (units holding an
|
|
916
|
+
* invocation and their ancestors' headers) are text; everything else is
|
|
917
|
+
* blanked with its line breaks kept, so every line keeps its number and the
|
|
918
|
+
* parser only walks what it must re-derive. Expansions replace their
|
|
919
|
+
* invocation's span.
|
|
920
|
+
*/
|
|
921
|
+
function buildSynthetic(content, regions, keep) {
|
|
922
|
+
let text = '';
|
|
923
|
+
let cursor = 0;
|
|
924
|
+
const spans = []; // synthetic [start, end) of text that is not blank
|
|
925
|
+
const blank = (from, to) => content.slice(from, to).replace(/[^\r\n]/g, ' ');
|
|
926
|
+
const addSpan = (start, end) => {
|
|
927
|
+
if (end <= start) return;
|
|
928
|
+
const last = spans[spans.length - 1];
|
|
929
|
+
if (last && last[1] >= start) last[1] = Math.max(last[1], end);
|
|
930
|
+
else spans.push([start, end]);
|
|
931
|
+
};
|
|
932
|
+
const copy = (from, to) => {
|
|
933
|
+
// Kept ranges are sorted and disjoint.
|
|
934
|
+
let at = from;
|
|
935
|
+
for (const [start, end] of keep) {
|
|
936
|
+
if (end <= at) continue;
|
|
937
|
+
if (start >= to) break;
|
|
938
|
+
if (start > at) text += blank(at, start);
|
|
939
|
+
const stop = Math.min(end, to);
|
|
940
|
+
const synthStart = text.length;
|
|
941
|
+
text += content.slice(Math.max(start, at), stop);
|
|
942
|
+
addSpan(synthStart, text.length);
|
|
943
|
+
at = stop;
|
|
944
|
+
}
|
|
945
|
+
if (at < to) text += blank(at, to);
|
|
946
|
+
};
|
|
947
|
+
for (const region of regions) {
|
|
948
|
+
copy(cursor, region.start);
|
|
949
|
+
region.synthStart = text.length;
|
|
950
|
+
text += region.layout.text;
|
|
951
|
+
region.synthEnd = text.length;
|
|
952
|
+
addSpan(region.synthStart, region.synthEnd);
|
|
953
|
+
cursor = region.end;
|
|
954
|
+
}
|
|
955
|
+
copy(cursor, content.length);
|
|
956
|
+
return { text, spans };
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
/**
|
|
960
|
+
* Parse only the non-blank spans of the synthetic file (tree-sitter
|
|
961
|
+
* included ranges): positions stay absolute, blank text costs nothing. The
|
|
962
|
+
* tree lands in safeParse's cache, so the adapter's analysis reuses it.
|
|
963
|
+
*/
|
|
964
|
+
function parseSynthetic(parser, text, spans) {
|
|
965
|
+
const starts = [0];
|
|
966
|
+
for (let i = text.indexOf('\n'); i >= 0; i = text.indexOf('\n', i + 1)) starts.push(i + 1);
|
|
967
|
+
const position = offset => {
|
|
968
|
+
let lo = 0;
|
|
969
|
+
let hi = starts.length - 1;
|
|
970
|
+
while (lo < hi) {
|
|
971
|
+
const mid = (lo + hi + 1) >> 1;
|
|
972
|
+
if (starts[mid] <= offset) lo = mid; else hi = mid - 1;
|
|
973
|
+
}
|
|
974
|
+
return { row: lo, column: offset - starts[lo] };
|
|
975
|
+
};
|
|
976
|
+
const includedRanges = spans.map(([start, end]) => ({
|
|
977
|
+
startIndex: start, endIndex: end, startPosition: position(start), endPosition: position(end),
|
|
978
|
+
}));
|
|
979
|
+
if (includedRanges.length === 0) return safeParse(parser, text, undefined, PARSE_OPTIONS);
|
|
980
|
+
return safeParse(parser, text, undefined, { ...PARSE_OPTIONS, includedRanges });
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
const UNIT_TYPES = new Set(['function_item', 'const_item', 'static_item']);
|
|
984
|
+
const HEADER_TYPES = new Set(['function_item', 'impl_item', 'trait_item', 'mod_item', 'foreign_mod_item']);
|
|
985
|
+
|
|
986
|
+
/** The node plus its preceding attributes and comments. */
|
|
987
|
+
function withLeadingTrivia(node) {
|
|
988
|
+
let start = node.startIndex;
|
|
989
|
+
let startRow = node.startPosition.row;
|
|
990
|
+
for (let sibling = node.previousSibling; sibling; sibling = sibling.previousSibling) {
|
|
991
|
+
if (sibling.type !== 'attribute_item' && sibling.type !== 'line_comment' &&
|
|
992
|
+
sibling.type !== 'block_comment') break;
|
|
993
|
+
start = sibling.startIndex;
|
|
994
|
+
startRow = sibling.startPosition.row;
|
|
995
|
+
}
|
|
996
|
+
return { start, startLine: startRow + 1 };
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* Units re-derived from the expanded file: the innermost function (or
|
|
1001
|
+
* const/static item) holding each invocation, else the item-position
|
|
1002
|
+
* invocation itself, with leading attributes; plus, kept as text, the
|
|
1003
|
+
* headers and associated types of every enclosing impl/trait/mod/fn so the
|
|
1004
|
+
* unit parses in its real context (Self type, trait, enclosing fn).
|
|
1005
|
+
*/
|
|
1006
|
+
function analysisUnits(regions, mode = 'calls') {
|
|
1007
|
+
const units = [];
|
|
1008
|
+
const keep = [];
|
|
1009
|
+
const headers = new Set();
|
|
1010
|
+
const keepHeader = parent => {
|
|
1011
|
+
if (headers.has(parent.startIndex)) return;
|
|
1012
|
+
headers.add(parent.startIndex);
|
|
1013
|
+
const body = parent.childForFieldName('body');
|
|
1014
|
+
if (!body) return;
|
|
1015
|
+
keep.push([parent.startIndex, body.startIndex + 1], [body.endIndex - 1, body.endIndex]);
|
|
1016
|
+
// Associated types of an enclosing impl/trait.
|
|
1017
|
+
if (parent.type === 'impl_item' || parent.type === 'trait_item') {
|
|
1018
|
+
for (const member of body.namedChildren) {
|
|
1019
|
+
if (member.type === 'type_item' || member.type === 'associated_type') {
|
|
1020
|
+
keep.push([member.startIndex, member.endIndex]);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
const { start: headStart } = withLeadingTrivia(parent);
|
|
1025
|
+
if (headStart < parent.startIndex) keep.push([headStart, parent.startIndex]);
|
|
1026
|
+
};
|
|
1027
|
+
for (const region of regions) {
|
|
1028
|
+
let unitNode = null;
|
|
1029
|
+
for (let parent = region.node.parent; parent; parent = parent.parent) {
|
|
1030
|
+
if (UNIT_TYPES.has(parent.type)) { unitNode = parent; break; }
|
|
1031
|
+
}
|
|
1032
|
+
let start;
|
|
1033
|
+
let startLine;
|
|
1034
|
+
let end;
|
|
1035
|
+
let endLine;
|
|
1036
|
+
if (unitNode) {
|
|
1037
|
+
({ start, startLine } = withLeadingTrivia(unitNode));
|
|
1038
|
+
end = unitNode.endIndex;
|
|
1039
|
+
endLine = unitNode.endPosition.row + 1;
|
|
1040
|
+
} else {
|
|
1041
|
+
({ start, startLine } = withLeadingTrivia(region.node));
|
|
1042
|
+
const next = region.node.nextSibling;
|
|
1043
|
+
const last = next && next.type === ';' ? next : region.node;
|
|
1044
|
+
end = last.endIndex;
|
|
1045
|
+
endLine = last.endPosition.row + 1;
|
|
1046
|
+
}
|
|
1047
|
+
units.push({ start, end, startLine, endLine });
|
|
1048
|
+
if (mode === 'symbols' && unitNode) {
|
|
1049
|
+
// Declarations need only the invocation in its enclosing
|
|
1050
|
+
// function's frame, not the rest of the body.
|
|
1051
|
+
if (unitNode.type === 'function_item') keepHeader(unitNode);
|
|
1052
|
+
else keep.push([start, end]);
|
|
1053
|
+
const next = region.node.nextSibling;
|
|
1054
|
+
keep.push([region.start, next && next.type === ';' ? next.endIndex : region.end]);
|
|
1055
|
+
} else {
|
|
1056
|
+
keep.push([start, end]);
|
|
1057
|
+
}
|
|
1058
|
+
for (let parent = (unitNode || region.node).parent; parent; parent = parent.parent) {
|
|
1059
|
+
if (HEADER_TYPES.has(parent.type)) keepHeader(parent);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
const merge = ranges => {
|
|
1063
|
+
ranges.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
1064
|
+
const out = [];
|
|
1065
|
+
for (const range of ranges) {
|
|
1066
|
+
const last = out[out.length - 1];
|
|
1067
|
+
if (last && range[0] <= last[1]) last[1] = Math.max(last[1], range[1]);
|
|
1068
|
+
else out.push([...range]);
|
|
1069
|
+
}
|
|
1070
|
+
return out;
|
|
1071
|
+
};
|
|
1072
|
+
units.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
1073
|
+
const merged = [];
|
|
1074
|
+
for (const unit of units) {
|
|
1075
|
+
const last = merged[merged.length - 1];
|
|
1076
|
+
if (last && unit.start <= last.end) {
|
|
1077
|
+
if (unit.end > last.end) { last.end = unit.end; last.endLine = unit.endLine; }
|
|
1078
|
+
} else merged.push({ ...unit });
|
|
1079
|
+
}
|
|
1080
|
+
return { units: merged, keep: merge(keep) };
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
function regionAt(regions, synth) {
|
|
1084
|
+
let lo = 0;
|
|
1085
|
+
let hi = regions.length - 1;
|
|
1086
|
+
while (lo <= hi) {
|
|
1087
|
+
const mid = (lo + hi) >> 1;
|
|
1088
|
+
const region = regions[mid];
|
|
1089
|
+
if (synth < region.synthStart) hi = mid - 1;
|
|
1090
|
+
else if (synth >= region.synthEnd) lo = mid + 1;
|
|
1091
|
+
else return region;
|
|
1092
|
+
}
|
|
1093
|
+
return null;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* Mark of the leaf at relative offset `rel`: the leaf containing it, else
|
|
1098
|
+
* (whitespace) the next leaf for a start offset, the previous for an end.
|
|
1099
|
+
*/
|
|
1100
|
+
function markAt(marks, rel, forEnd) {
|
|
1101
|
+
const probe = forEnd ? rel - 1 : rel;
|
|
1102
|
+
let lo = 0;
|
|
1103
|
+
let hi = marks.length - 1;
|
|
1104
|
+
while (lo <= hi) {
|
|
1105
|
+
const mid = (lo + hi) >> 1;
|
|
1106
|
+
const mark = marks[mid];
|
|
1107
|
+
if (probe < mark.s) hi = mid - 1;
|
|
1108
|
+
else if (probe >= mark.e) lo = mid + 1;
|
|
1109
|
+
else return mark;
|
|
1110
|
+
}
|
|
1111
|
+
// lo = first mark starting after probe, hi = last mark ending before it
|
|
1112
|
+
return forEnd ? (marks[hi] || marks[lo] || null) : (marks[lo] || marks[hi] || null);
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
/** Maps synthetic offsets back to the source file. */
|
|
1116
|
+
function offsetMapper(regions) {
|
|
1117
|
+
// Length change accumulated after each region.
|
|
1118
|
+
let delta = 0;
|
|
1119
|
+
for (const region of regions) {
|
|
1120
|
+
delta += (region.synthEnd - region.synthStart) - (region.end - region.start);
|
|
1121
|
+
region.deltaAfter = delta;
|
|
1122
|
+
}
|
|
1123
|
+
const map = (synth, forEnd) => {
|
|
1124
|
+
const region = regionAt(regions, forEnd ? synth - 1 : synth);
|
|
1125
|
+
if (!region) {
|
|
1126
|
+
// Copied text: shift by every region before it.
|
|
1127
|
+
let lo = 0;
|
|
1128
|
+
let hi = regions.length - 1;
|
|
1129
|
+
let before = null;
|
|
1130
|
+
while (lo <= hi) {
|
|
1131
|
+
const mid = (lo + hi) >> 1;
|
|
1132
|
+
if (regions[mid].synthEnd <= synth) { before = regions[mid]; lo = mid + 1; } else hi = mid - 1;
|
|
1133
|
+
}
|
|
1134
|
+
return synth - (before ? before.deltaAfter : 0);
|
|
1135
|
+
}
|
|
1136
|
+
const rel = synth - region.synthStart;
|
|
1137
|
+
const mark = markAt(region.layout.marks, rel, forEnd);
|
|
1138
|
+
if (mark && mark.leaf.src === 'arg' && Number.isInteger(mark.leaf.s)) {
|
|
1139
|
+
return mark.leaf.s + Math.min(Math.max(rel - mark.s, 0), mark.e - mark.s);
|
|
1140
|
+
}
|
|
1141
|
+
return fraction(region, synth);
|
|
1142
|
+
};
|
|
1143
|
+
// Template text has no source offset: a stable fraction inside the
|
|
1144
|
+
// invocation's first byte keeps order and receiver links.
|
|
1145
|
+
const fraction = (region, synth) =>
|
|
1146
|
+
region.start + (synth - region.synthStart + 1) / (region.synthEnd - region.synthStart + 2);
|
|
1147
|
+
const templateOffset = synth => {
|
|
1148
|
+
const region = regionAt(regions, synth) || regionAt(regions, synth - 1);
|
|
1149
|
+
return region ? fraction(region, Math.min(synth, region.synthEnd)) : map(synth, false);
|
|
1150
|
+
};
|
|
1151
|
+
return { map, templateOffset, regionAt: synth => regionAt(regions, synth) };
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
const OFFSET_FIELD = /(?:Start|End|^start|^end)$/;
|
|
1155
|
+
|
|
1156
|
+
/**
|
|
1157
|
+
* Remap every `<x>Start`/`<x>End` offset field. A span keeps source offsets
|
|
1158
|
+
* only when both ends were written in the source (outside expansions, or
|
|
1159
|
+
* both inside argument tokens); otherwise both ends take the invocation's
|
|
1160
|
+
* fractional template offsets, so a span never inverts.
|
|
1161
|
+
*/
|
|
1162
|
+
function remapOffsets(record, mapper, depth = 0) {
|
|
1163
|
+
const pairs = new Map();
|
|
1164
|
+
for (const key of Object.keys(record)) {
|
|
1165
|
+
const value = record[key];
|
|
1166
|
+
if (value && typeof value === 'object' && depth < 4) {
|
|
1167
|
+
// Nested evidence (receiverTypeEvidence, origins) carries offsets too.
|
|
1168
|
+
if (Array.isArray(value)) {
|
|
1169
|
+
record[key] = value.map(item => {
|
|
1170
|
+
if (!item || typeof item !== 'object') return item;
|
|
1171
|
+
const copy = { ...item };
|
|
1172
|
+
remapOffsets(copy, mapper, depth + 1);
|
|
1173
|
+
return copy;
|
|
1174
|
+
});
|
|
1175
|
+
} else {
|
|
1176
|
+
const copy = { ...value };
|
|
1177
|
+
remapOffsets(copy, mapper, depth + 1);
|
|
1178
|
+
record[key] = copy;
|
|
1179
|
+
}
|
|
1180
|
+
continue;
|
|
1181
|
+
}
|
|
1182
|
+
if (typeof value !== 'number' || !OFFSET_FIELD.test(key)) continue;
|
|
1183
|
+
const stem = key.replace(OFFSET_FIELD, '');
|
|
1184
|
+
if (!pairs.has(stem)) pairs.set(stem, {});
|
|
1185
|
+
pairs.get(stem)[/end$/i.test(key) ? 'end' : 'start'] = key;
|
|
1186
|
+
}
|
|
1187
|
+
for (const { start, end } of pairs.values()) {
|
|
1188
|
+
const mappedStart = start ? mapper.map(record[start], false) : null;
|
|
1189
|
+
const mappedEnd = end ? mapper.map(record[end], true) : null;
|
|
1190
|
+
const mixed = start && end &&
|
|
1191
|
+
(Number.isInteger(mappedStart) !== Number.isInteger(mappedEnd) || mappedEnd < mappedStart);
|
|
1192
|
+
if (mixed) {
|
|
1193
|
+
record[start] = mapper.templateOffset(record[start]);
|
|
1194
|
+
record[end] = mapper.templateOffset(record[end]);
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
if (start) record[start] = mappedStart;
|
|
1198
|
+
if (end) record[end] = mappedEnd;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
const DECLARATION_TYPES = [
|
|
1203
|
+
'function_item', 'function_signature_item', 'impl_item', 'struct_item', 'enum_item', 'union_item',
|
|
1204
|
+
'trait_item', 'mod_item', 'type_item', 'const_item', 'static_item', 'macro_definition',
|
|
1205
|
+
'field_declaration', 'enum_variant', 'associated_type',
|
|
1206
|
+
];
|
|
1207
|
+
|
|
1208
|
+
/** (startLine\0name) keys of declarations whose name was produced inside a region. */
|
|
1209
|
+
function generatedDeclarations(root, regions) {
|
|
1210
|
+
const out = new Map();
|
|
1211
|
+
if (regions.length === 0) return out;
|
|
1212
|
+
for (const node of root.descendantsOfType(DECLARATION_TYPES)) {
|
|
1213
|
+
const nameNode = node.type === 'impl_item' ? node.childForFieldName('type')
|
|
1214
|
+
: node.childForFieldName('name');
|
|
1215
|
+
if (!nameNode) continue;
|
|
1216
|
+
const region = regionAt(regions, nameNode.startIndex);
|
|
1217
|
+
if (!region || nameNode.endIndex > region.synthEnd) continue;
|
|
1218
|
+
const mark = markAt(region.layout.marks, nameNode.startIndex - region.synthStart, false);
|
|
1219
|
+
const origin = mark?.leaf.src === 'arg' ? 'argument' : 'template';
|
|
1220
|
+
const text = node.type === 'impl_item' ? typeNameOf(nameNode) : nameNode.text;
|
|
1221
|
+
out.set(`${node.startPosition.row + 1}\0${text}`, {
|
|
1222
|
+
region, origin,
|
|
1223
|
+
...(origin === 'argument' && { nameLine: mark.leaf.line }),
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1226
|
+
return out;
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
function typeNameOf(node) {
|
|
1230
|
+
if (node.type === 'generic_type') return typeNameOf(node.childForFieldName('type'));
|
|
1231
|
+
if (node.type === 'scoped_type_identifier') return node.childForFieldName('name')?.text || node.text;
|
|
1232
|
+
if (node.type === 'reference_type') return typeNameOf(node.childForFieldName('type'));
|
|
1233
|
+
return node.text;
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
function macroFact(region, origin) {
|
|
1237
|
+
return {
|
|
1238
|
+
macro: region.macro,
|
|
1239
|
+
definition: region.definition,
|
|
1240
|
+
origin,
|
|
1241
|
+
line: region.startLine,
|
|
1242
|
+
...(region.ambiguous && { ambiguous: true }),
|
|
1243
|
+
};
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
/**
|
|
1247
|
+
* Expand the project macro invocations of one file. Returns the expansion
|
|
1248
|
+
* record, or null when the file invokes no project macro in scope.
|
|
1249
|
+
*/
|
|
1250
|
+
function expandFile(state, filePath, mode = 'symbols') {
|
|
1251
|
+
// Hygiene contexts are numbered per file, so an expansion's renamed
|
|
1252
|
+
// locals do not depend on what else this state expanded before.
|
|
1253
|
+
state.hygieneSeq = 0;
|
|
1254
|
+
const index = state.index;
|
|
1255
|
+
const fileEntry = index.files.get(filePath);
|
|
1256
|
+
const content = readContent(state, filePath);
|
|
1257
|
+
if (!fileEntry || content === null) return null;
|
|
1258
|
+
if (!fileMayExpand(state, filePath)) return { sites: 0, blind: [], used: new Map(), expanded: false };
|
|
1259
|
+
const tree = safeParse(state.parser, content, undefined, PARSE_OPTIONS);
|
|
1260
|
+
const rec = { sites: 0, blind: [], used: new Map(), expanded: false };
|
|
1261
|
+
const regions = [];
|
|
1262
|
+
let lineStarts = null;
|
|
1263
|
+
const invocations = tree.rootNode.descendantsOfType('macro_invocation');
|
|
1264
|
+
for (const node of invocations) {
|
|
1265
|
+
const id = invocationName(node);
|
|
1266
|
+
if (!id || !state.inventory.has(id.name)) continue;
|
|
1267
|
+
let inDefinition = false;
|
|
1268
|
+
for (let parent = node.parent; parent; parent = parent.parent) {
|
|
1269
|
+
if (parent.type === 'macro_definition') { inDefinition = true; break; }
|
|
1270
|
+
}
|
|
1271
|
+
if (inDefinition) continue;
|
|
1272
|
+
const line = node.startPosition.row + 1;
|
|
1273
|
+
const resolved = resolveMacro(state, filePath, node.startIndex, line, id.name, id.qualifier, null);
|
|
1274
|
+
if (!resolved) continue;
|
|
1275
|
+
const def = resolved.defs[0];
|
|
1276
|
+
const info = compiledDef(state, def);
|
|
1277
|
+
const context = invocationContext(node);
|
|
1278
|
+
if (!worthExpanding(state, resolved.defs, content.slice(node.startIndex, node.endIndex),
|
|
1279
|
+
context === 'expr')) continue;
|
|
1280
|
+
const tokenTree = node.namedChildren.find(child => child.type === 'token_tree');
|
|
1281
|
+
if (info.compiled?.passThroughItems && context === 'items' && itemRecoveryPosition(node)) continue;
|
|
1282
|
+
const blind = reason => rec.blind.push({ line, macro: id.name, reason });
|
|
1283
|
+
if (!context) { blind('context'); continue; }
|
|
1284
|
+
if (!tokenTree) { blind('token-error'); continue; }
|
|
1285
|
+
const site = {
|
|
1286
|
+
file: filePath, offset: node.startIndex, line, budget: MAX_OUTPUT_TOKENS,
|
|
1287
|
+
used: rec.used, nestedBlind: [], ambiguous: resolved.ambiguous,
|
|
1288
|
+
};
|
|
1289
|
+
let tokens;
|
|
1290
|
+
try {
|
|
1291
|
+
if (!lineStarts) {
|
|
1292
|
+
lineOffset(state, filePath, 1);
|
|
1293
|
+
lineStarts = state.lineStarts.get(filePath);
|
|
1294
|
+
}
|
|
1295
|
+
const toks = argTokensOf(tokenTree, content, lineStarts);
|
|
1296
|
+
tokens = resolved.alternatives
|
|
1297
|
+
? expandAlternatives(state, site, resolved, toks, 0, context)
|
|
1298
|
+
: expandTokens(state, site, def, toks, 0);
|
|
1299
|
+
} catch (error) {
|
|
1300
|
+
if (!(error instanceof MacroBlind)) throw error;
|
|
1301
|
+
blind(error.reason);
|
|
1302
|
+
continue;
|
|
1303
|
+
}
|
|
1304
|
+
for (const nested of site.nestedBlind) rec.blind.push({ line, macro: nested.macro, reason: `nested:${nested.reason}` });
|
|
1305
|
+
renameHygienicLocals(tokens);
|
|
1306
|
+
let body = tokens;
|
|
1307
|
+
if (context === 'expr') body = wrapExpression(tokens);
|
|
1308
|
+
const startLine = line;
|
|
1309
|
+
const endLine = node.endPosition.row + 1;
|
|
1310
|
+
const ancestors = new Set();
|
|
1311
|
+
for (let parent = node.parent; parent; parent = parent.parent) {
|
|
1312
|
+
const nameNode = parent.type === 'impl_item' ? parent.childForFieldName('type')
|
|
1313
|
+
: DECLARATION_TYPES.includes(parent.type) ? parent.childForFieldName('name') : null;
|
|
1314
|
+
if (nameNode) {
|
|
1315
|
+
ancestors.add(`${parent.startPosition.row + 1}\0${parent.type === 'impl_item' ? typeNameOf(nameNode) : nameNode.text}`);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
regions.push({
|
|
1319
|
+
start: node.startIndex, end: node.endIndex, startLine, endLine,
|
|
1320
|
+
macro: id.name, definition: defKey(def), ambiguous: site.ambiguous, ancestors,
|
|
1321
|
+
node, layout: layout(body, startLine, node.startPosition.column, endLine),
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
rec.sites = regions.length;
|
|
1325
|
+
if (regions.length === 0) return rec;
|
|
1326
|
+
regions.sort((a, b) => a.start - b.start);
|
|
1327
|
+
// Every word the expanded text of the calls pass can hold (fix #375).
|
|
1328
|
+
const words = mode === 'symbols' ? expansionWords(content, regions) : null;
|
|
1329
|
+
|
|
1330
|
+
// Parse the expanded file; an expansion that does not parse in its
|
|
1331
|
+
// context stays as written (blind), and the rest are parsed again.
|
|
1332
|
+
let synthetic;
|
|
1333
|
+
let synthTree;
|
|
1334
|
+
let units;
|
|
1335
|
+
for (let attempt = 0; attempt <= regions.length && regions.length > 0; attempt++) {
|
|
1336
|
+
const scope = analysisUnits(regions, mode);
|
|
1337
|
+
units = scope.units;
|
|
1338
|
+
const built = buildSynthetic(content, regions, scope.keep);
|
|
1339
|
+
synthetic = built.text;
|
|
1340
|
+
synthTree = parseSynthetic(state.parser, synthetic, built.spans);
|
|
1341
|
+
const errors = collectErrorRanges(synthTree.rootNode, []);
|
|
1342
|
+
const bad = new Set();
|
|
1343
|
+
for (const [start, end] of errors) {
|
|
1344
|
+
for (const region of regions) {
|
|
1345
|
+
if (start < region.synthEnd && end > region.synthStart) bad.add(region);
|
|
1346
|
+
else if (start === end && start >= region.synthStart && start <= region.synthEnd) bad.add(region);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
if (bad.size === 0) break;
|
|
1350
|
+
for (const region of bad) {
|
|
1351
|
+
rec.blind.push({ line: region.startLine, macro: region.macro, reason: 'parse-error' });
|
|
1352
|
+
regions.splice(regions.indexOf(region), 1);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
rec.sites = regions.length;
|
|
1356
|
+
if (regions.length === 0) return rec;
|
|
1357
|
+
|
|
1358
|
+
const rust = require('../languages/rust');
|
|
1359
|
+
// The expanded file holds no unexpanded item bodies worth recovering
|
|
1360
|
+
// inside its units; the extractors share the included-ranges tree.
|
|
1361
|
+
rust.primeDeclarationTrees(synthetic, state.parser, synthTree);
|
|
1362
|
+
const mapper = offsetMapper(regions);
|
|
1363
|
+
|
|
1364
|
+
// Units are re-derived; everything outside them keeps its parse.
|
|
1365
|
+
// (Sorted and disjoint: binary searches.)
|
|
1366
|
+
const unitAtOffset = offset => {
|
|
1367
|
+
let lo = 0;
|
|
1368
|
+
let hi = units.length - 1;
|
|
1369
|
+
while (lo <= hi) {
|
|
1370
|
+
const mid = (lo + hi) >> 1;
|
|
1371
|
+
if (offset < units[mid].start) hi = mid - 1;
|
|
1372
|
+
else if (offset >= units[mid].end) lo = mid + 1;
|
|
1373
|
+
else return units[mid];
|
|
1374
|
+
}
|
|
1375
|
+
return null;
|
|
1376
|
+
};
|
|
1377
|
+
const byLine = (list, line) => {
|
|
1378
|
+
let lo = 0;
|
|
1379
|
+
let hi = list.length - 1;
|
|
1380
|
+
while (lo <= hi) {
|
|
1381
|
+
const mid = (lo + hi) >> 1;
|
|
1382
|
+
if (line < list[mid].startLine) hi = mid - 1;
|
|
1383
|
+
else if (line > list[mid].endLine) lo = mid + 1;
|
|
1384
|
+
else return list[mid];
|
|
1385
|
+
}
|
|
1386
|
+
return null;
|
|
1387
|
+
};
|
|
1388
|
+
const unitAtLine = line => byLine(units, line);
|
|
1389
|
+
const inUnit = record => (Number.isFinite(record.callStart) ? !!unitAtOffset(record.callStart)
|
|
1390
|
+
: !!unitAtLine(record.line));
|
|
1391
|
+
rec.expanded = true;
|
|
1392
|
+
|
|
1393
|
+
if (mode === 'symbols') {
|
|
1394
|
+
// Declarations of the regions only: the smallest nodes spanning each.
|
|
1395
|
+
const spanning = regions.map(region => (region.synthEnd > region.synthStart
|
|
1396
|
+
? synthTree.rootNode.descendantForIndex(region.synthStart, region.synthEnd - 1) : null))
|
|
1397
|
+
.filter(Boolean).sort((a, b) => a.startIndex - b.startIndex || b.endIndex - a.endIndex);
|
|
1398
|
+
const roots = [];
|
|
1399
|
+
for (const node of spanning) {
|
|
1400
|
+
const last = roots[roots.length - 1];
|
|
1401
|
+
if (last && node.startIndex >= last.startIndex && node.endIndex <= last.endIndex) continue;
|
|
1402
|
+
roots.push(node);
|
|
1403
|
+
}
|
|
1404
|
+
rec.symbols = generatedSymbols(filePath, rust.parseDeclarationsIn(synthetic, roots), {
|
|
1405
|
+
regions, mapper, generated: generatedDeclarations(synthTree.rootNode, regions),
|
|
1406
|
+
});
|
|
1407
|
+
rec.calleeNames = callableNames(regions);
|
|
1408
|
+
rec.words = words;
|
|
1409
|
+
rec.callsPending = true;
|
|
1410
|
+
return rec;
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
// Calls: records of the units from the expanded file, the rest as parsed,
|
|
1414
|
+
// plus the invocations themselves.
|
|
1415
|
+
const cached = index.callsCache.get(filePath);
|
|
1416
|
+
const original = cached?.calls || [];
|
|
1417
|
+
const invocationStarts = new Set(regions.map(region => region.start));
|
|
1418
|
+
const calls = original.filter(record => !inUnit(record));
|
|
1419
|
+
const seen = new Set();
|
|
1420
|
+
// Records without a source span (struct-literal constructors) are placed
|
|
1421
|
+
// by line: one the unexpanded parse also has on that line is ordinary
|
|
1422
|
+
// code, any other one on an invocation's lines was generated.
|
|
1423
|
+
const spanless = record => `${record.name}\0${record.line}\0${record.isConstructor ? 1 : 0}`;
|
|
1424
|
+
const originalSpanless = new Map();
|
|
1425
|
+
for (const record of original) {
|
|
1426
|
+
if (Number.isFinite(record.callStart) || !unitAtLine(record.line)) continue;
|
|
1427
|
+
const key = spanless(record);
|
|
1428
|
+
originalSpanless.set(key, (originalSpanless.get(key) || 0) + 1);
|
|
1429
|
+
}
|
|
1430
|
+
const regionOnLine = line => byLine(regions, line);
|
|
1431
|
+
const synthCalls = rust.findCallsInCode(synthetic, state.parser);
|
|
1432
|
+
for (const record of synthCalls) {
|
|
1433
|
+
const synthStart = record.callStart;
|
|
1434
|
+
if (Number.isFinite(synthStart)) {
|
|
1435
|
+
const mapped = mapper.map(synthStart, false);
|
|
1436
|
+
if (!unitAtOffset(Math.floor(mapped))) continue;
|
|
1437
|
+
} else if (!unitAtLine(record.line)) continue;
|
|
1438
|
+
const call = { ...record };
|
|
1439
|
+
let region = Number.isFinite(synthStart) ? mapper.regionAt(synthStart) : null;
|
|
1440
|
+
if (!Number.isFinite(synthStart) && regionOnLine(record.line)) {
|
|
1441
|
+
const key = spanless(record);
|
|
1442
|
+
const left = originalSpanless.get(key) || 0;
|
|
1443
|
+
if (left > 0) originalSpanless.set(key, left - 1);
|
|
1444
|
+
else region = regionOnLine(record.line);
|
|
1445
|
+
}
|
|
1446
|
+
remapOffsets(call, mapper);
|
|
1447
|
+
if (region) {
|
|
1448
|
+
// Origin of the NAME token: `v.iter().collect()` written with a
|
|
1449
|
+
// receiver from the arguments still calls a template `collect`.
|
|
1450
|
+
const nameMark = Number.isFinite(synthStart) ? nameMarkOf(synthTree.rootNode, region, record)
|
|
1451
|
+
: region.layout.marks.find(mark => mark.leaf.v === record.name && mark.leaf.src === 'arg' &&
|
|
1452
|
+
mark.line === record.line) || null;
|
|
1453
|
+
const origin = nameMark?.leaf.src === 'arg' ? 'argument' : 'template';
|
|
1454
|
+
// An argument token the layout could not keep on its own line
|
|
1455
|
+
// (repeated or reordered by the transcriber) reports its own.
|
|
1456
|
+
if (origin === 'argument' && nameMark.line !== nameMark.leaf.line) call.line = nameMark.leaf.line;
|
|
1457
|
+
call.macroExpansion = macroFact(region, origin);
|
|
1458
|
+
// Repeated argument tokens are one call; template calls at one
|
|
1459
|
+
// line are told apart only by what the engine can see of them
|
|
1460
|
+
// (a transcriber repetition emits the same call per iteration).
|
|
1461
|
+
const key = origin === 'argument'
|
|
1462
|
+
? `a\0${call.name}\0${call.line}\0${call.callStart}\0${call.callEnd}\0${call.receiver || ''}`
|
|
1463
|
+
: `t\0${call.name}\0${call.line}\0${call.isMethod ? 1 : 0}\0${call.receiver || ''}\0` +
|
|
1464
|
+
`${call.receiverType || ''}\0${call.receiverRootType || ''}\0${call.receiverField || ''}\0` +
|
|
1465
|
+
`${call.enclosingFunction?.name || ''}\0${call.argCount ?? ''}`;
|
|
1466
|
+
if (seen.has(key)) continue;
|
|
1467
|
+
seen.add(key);
|
|
1468
|
+
}
|
|
1469
|
+
calls.push(call);
|
|
1470
|
+
}
|
|
1471
|
+
// The invocation stays a call of the macro; the value it produces is
|
|
1472
|
+
// the expansion's, so its own value-flow facts are dropped.
|
|
1473
|
+
for (const record of original) {
|
|
1474
|
+
if (!record.isMacro || !invocationStarts.has(record.callStart)) continue;
|
|
1475
|
+
calls.push({
|
|
1476
|
+
name: record.name, line: record.line, callStart: record.callStart, callEnd: record.callEnd,
|
|
1477
|
+
isMethod: false, isMacro: true,
|
|
1478
|
+
...(record.receiver && { receiver: record.receiver, isPathMacro: true }),
|
|
1479
|
+
...(record.macroExpr && { macroExpr: true }),
|
|
1480
|
+
enclosingFunction: record.enclosingFunction,
|
|
1481
|
+
macroExpanded: true,
|
|
1482
|
+
});
|
|
1483
|
+
}
|
|
1484
|
+
// Shapes of generated calls whose value may be lost (audit-async).
|
|
1485
|
+
const { openCallShape } = require('../languages/rust-value-flow');
|
|
1486
|
+
const shapes = calls.filter(call => call.macroExpansion && call.name && !call.inMacro && !call.valueConsumed)
|
|
1487
|
+
.map(openCallShape);
|
|
1488
|
+
if (shapes.length > 0) {
|
|
1489
|
+
fileEntry.openCalls = [...new Set([...(fileEntry.openCalls || []), ...shapes])].sort(codeUnitCompare);
|
|
1490
|
+
}
|
|
1491
|
+
// Stable by line: records keep document order within a line.
|
|
1492
|
+
calls.sort((a, b) => a.line - b.line);
|
|
1493
|
+
if (cached) {
|
|
1494
|
+
if (index.calleeIndex) index._removeFromCalleeIndex(filePath, original);
|
|
1495
|
+
cached.calls = calls;
|
|
1496
|
+
if (index.calleeIndex) index._addToCalleeIndex(filePath, calls);
|
|
1497
|
+
} else {
|
|
1498
|
+
index.callsCache.set(filePath, { mtime: fileEntry.mtime, hash: fileEntry.hash, calls });
|
|
1499
|
+
}
|
|
1500
|
+
index.callsCacheDirty = true;
|
|
1501
|
+
rec.expanded = true;
|
|
1502
|
+
return rec;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
const CALL_NODES = new Set(['call_expression', 'macro_invocation', 'struct_expression', 'generic_function']);
|
|
1506
|
+
|
|
1507
|
+
/** Layout mark of the name token of a call record inside a region. */
|
|
1508
|
+
function nameMarkOf(root, region, record) {
|
|
1509
|
+
const start = record.callStart;
|
|
1510
|
+
const end = record.callEnd;
|
|
1511
|
+
let node = Number.isFinite(start) && Number.isFinite(end) && end > start
|
|
1512
|
+
? root.descendantForIndex(start, end - 1) : null;
|
|
1513
|
+
while (node && !(CALL_NODES.has(node.type) && node.startIndex === start && node.endIndex === end)) {
|
|
1514
|
+
node = node.parent;
|
|
1515
|
+
}
|
|
1516
|
+
let nameNode = null;
|
|
1517
|
+
if (node?.type === 'macro_invocation') {
|
|
1518
|
+
nameNode = node.childForFieldName('macro');
|
|
1519
|
+
} else if (node) {
|
|
1520
|
+
let fn = node.childForFieldName('function') || node.childForFieldName('name');
|
|
1521
|
+
if (fn?.type === 'generic_function') fn = fn.childForFieldName('function');
|
|
1522
|
+
if (fn?.type === 'field_expression') nameNode = fn.childForFieldName('field');
|
|
1523
|
+
else if (fn?.type === 'scoped_identifier' || fn?.type === 'scoped_type_identifier') nameNode = fn.childForFieldName('name');
|
|
1524
|
+
else nameNode = fn;
|
|
1525
|
+
}
|
|
1526
|
+
if (nameNode && nameNode.startIndex >= region.synthStart && nameNode.endIndex <= region.synthEnd) {
|
|
1527
|
+
const rel = nameNode.type === 'scoped_identifier' || nameNode.type === 'macro_invocation'
|
|
1528
|
+
? nameNode.endIndex - 1 - region.synthStart : nameNode.startIndex - region.synthStart;
|
|
1529
|
+
return markAt(region.layout.marks, rel, false);
|
|
1530
|
+
}
|
|
1531
|
+
// No node at that span: the first leaf spelling the name.
|
|
1532
|
+
const marks = region.layout.marks;
|
|
1533
|
+
return marks.find(mark => mark.leaf.v === record.name &&
|
|
1534
|
+
mark.s >= start - region.synthStart && mark.e <= end - region.synthStart) || null;
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
/**
|
|
1538
|
+
* The declarations the expansions generate (IR items carrying
|
|
1539
|
+
* `macroExpansion`), and the invocation spans whose recovered declarations
|
|
1540
|
+
* they replace. Pure: computed in a build worker or in place.
|
|
1541
|
+
*/
|
|
1542
|
+
function generatedSymbols(filePath, parsed, { regions, mapper, generated }) {
|
|
1543
|
+
const { createFileIR } = require('./ir');
|
|
1544
|
+
const ir = createFileIR({ language: 'rust', file: filePath, parsed, calls: [] });
|
|
1545
|
+
const symbols = [];
|
|
1546
|
+
const seen = new Set();
|
|
1547
|
+
let ordinal = 0;
|
|
1548
|
+
for (const item of ir.symbols) {
|
|
1549
|
+
const origin = generated.get(`${item.startLine}\0${item.name}`);
|
|
1550
|
+
if (!origin) continue;
|
|
1551
|
+
// Declarations a repetition generates identically at one line.
|
|
1552
|
+
const key = `${item.kind}\0${item.name}\0${item.owner || ''}\0${item.startLine}\0${item.traitName || ''}\0` +
|
|
1553
|
+
`${item.params || ''}\0${item.returnType || ''}`;
|
|
1554
|
+
if (seen.has(key)) continue;
|
|
1555
|
+
seen.add(key);
|
|
1556
|
+
const symbol = { ...item };
|
|
1557
|
+
remapOffsets(symbol, mapper);
|
|
1558
|
+
symbol.macroExpansion = macroFact(origin.region, origin.origin);
|
|
1559
|
+
if (origin.nameLine && origin.nameLine !== symbol.startLine) symbol.nameLine = origin.nameLine;
|
|
1560
|
+
symbol.id = `macro:${symbol.startLine}:${symbol.kind}:${symbol.owner || ''}:${symbol.name}:${ordinal++}`;
|
|
1561
|
+
symbols.push(symbol);
|
|
1562
|
+
}
|
|
1563
|
+
return {
|
|
1564
|
+
symbols,
|
|
1565
|
+
spans: regions.map(region => ({
|
|
1566
|
+
startLine: region.startLine, endLine: region.endLine, ancestors: [...region.ancestors],
|
|
1567
|
+
})),
|
|
1568
|
+
};
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
/** Apply generatedSymbols() to the index (main thread). */
|
|
1572
|
+
function applyGeneratedSymbols(index, filePath, result) {
|
|
1573
|
+
const fileEntry = index.files.get(filePath);
|
|
1574
|
+
if (!fileEntry || !result) return;
|
|
1575
|
+
for (const symbol of fileEntry.symbols) {
|
|
1576
|
+
const entries = index.symbols.get(symbol.name);
|
|
1577
|
+
if (!entries) continue;
|
|
1578
|
+
const kept = entries.filter(entry => entry.file !== filePath);
|
|
1579
|
+
if (kept.length > 0) index.symbols.set(symbol.name, kept);
|
|
1580
|
+
else index.symbols.delete(symbol.name);
|
|
1581
|
+
}
|
|
1582
|
+
const previous = fileEntry.symbols;
|
|
1583
|
+
fileEntry.symbols = [];
|
|
1584
|
+
fileEntry.bindings = [];
|
|
1585
|
+
// An invocation's token trees held only what the parser recovered from
|
|
1586
|
+
// them (item bodies): those give way to what the expansion declares.
|
|
1587
|
+
// The declarations enclosing an invocation stay.
|
|
1588
|
+
const spans = result.spans.map(span => ({ ...span, ancestors: new Set(span.ancestors) }));
|
|
1589
|
+
const insideSpan = symbol => spans.some(span => symbol.startLine >= span.startLine &&
|
|
1590
|
+
symbol.endLine <= span.endLine && !span.ancestors.has(`${symbol.startLine}\0${symbol.name}`));
|
|
1591
|
+
for (const symbol of previous) {
|
|
1592
|
+
if (insideSpan(symbol)) continue;
|
|
1593
|
+
fileEntry.symbols.push(symbol);
|
|
1594
|
+
if (!index.symbols.has(symbol.name)) index.symbols.set(symbol.name, []);
|
|
1595
|
+
index.symbols.get(symbol.name).push(symbol);
|
|
1596
|
+
if (symbol.memberAssigned || symbol.bodyScopedName || symbol.exportedAlias || symbol.type === 'impl') continue;
|
|
1597
|
+
fileEntry.bindings.push({
|
|
1598
|
+
id: symbol.bindingId, name: symbol.name, type: symbol.type, startLine: symbol.startLine,
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1601
|
+
for (const item of result.symbols) {
|
|
1602
|
+
const symbol = addIRSymbol(fileEntry, item, index.symbols);
|
|
1603
|
+
// A generated trait impl for a type declared in this file joins the
|
|
1604
|
+
// type's `implements` (the parser does this for impls it sees).
|
|
1605
|
+
if (symbol.type !== 'impl' || !symbol.traitName) continue;
|
|
1606
|
+
const owner = fileEntry.symbols.find(candidate => (candidate.type === 'struct' || candidate.type === 'enum') &&
|
|
1607
|
+
candidate.name === symbol.name && !candidate.macroExpansion);
|
|
1608
|
+
if (owner && !(owner.implements || []).includes(symbol.traitName)) {
|
|
1609
|
+
owner.implements = [...(owner.implements || []), symbol.traitName];
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
/**
|
|
1615
|
+
* Names the calls of the expansions can bear (callee position, method,
|
|
1616
|
+
* macro, struct literal): the callee index lists the file under them until
|
|
1617
|
+
* its calls are expanded.
|
|
1618
|
+
*/
|
|
1619
|
+
function callableNames(regions) {
|
|
1620
|
+
const names = new Set();
|
|
1621
|
+
for (const region of regions) {
|
|
1622
|
+
const marks = region.layout.marks;
|
|
1623
|
+
for (let i = 0; i < marks.length - 1; i++) {
|
|
1624
|
+
const leaf = marks[i].leaf;
|
|
1625
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(leaf.v) || RUST_KEYWORDS.has(leaf.v)) continue;
|
|
1626
|
+
const next = marks[i + 1].leaf.v;
|
|
1627
|
+
if (next === '(' || next === '!' || next === '{' ||
|
|
1628
|
+
(next === '::' && marks[i + 2]?.leaf.v === '<')) names.add(leaf.v);
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
return [...names].sort(codeUnitCompare);
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
const WORD = /[\p{L}\p{N}_]+/gu;
|
|
1635
|
+
const WHOLE_WORD = /^[\p{L}\p{N}_]+$/u;
|
|
1636
|
+
|
|
1637
|
+
/**
|
|
1638
|
+
* The words (identifier-like runs) of the text the calls pass analyzes for
|
|
1639
|
+
* these regions: the kept source around them (units and their headers) and
|
|
1640
|
+
* every expansion's tokens. A call record derived from that text can bear no
|
|
1641
|
+
* other name, so a file whose calls are not derived yet needs deriving only
|
|
1642
|
+
* for a name among them (fix #375). Space-separated, sorted.
|
|
1643
|
+
*/
|
|
1644
|
+
function expansionWords(content, regions) {
|
|
1645
|
+
const words = new Set();
|
|
1646
|
+
for (const [start, end] of analysisUnits(regions, 'calls').keep) {
|
|
1647
|
+
for (const word of content.slice(start, end).match(WORD) || []) words.add(word);
|
|
1648
|
+
}
|
|
1649
|
+
// A record's name is an identifier token (string contents never are;
|
|
1650
|
+
// a non-ASCII identifier is a 'punct' leaf).
|
|
1651
|
+
for (const region of regions) {
|
|
1652
|
+
for (const mark of region.layout.marks) {
|
|
1653
|
+
const leaf = mark.leaf;
|
|
1654
|
+
if (leaf.k === 'ident' || (leaf.k === 'punct' && WHOLE_WORD.test(leaf.v))) words.add(leaf.v);
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
return [...words].sort(codeUnitCompare).join(' ');
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
/** Whether the derived calls of a file pending derivation can bear `name`. */
|
|
1661
|
+
const pendingWordSets = new WeakMap();
|
|
1662
|
+
function rustPendingMayBear(fileEntry, name) {
|
|
1663
|
+
const rec = fileEntry?.rustMacroExpansion;
|
|
1664
|
+
if (!rec?.callsPending) return false;
|
|
1665
|
+
if (typeof rec.words !== 'string') return true;
|
|
1666
|
+
let words = pendingWordSets.get(rec);
|
|
1667
|
+
if (!words) pendingWordSets.set(rec, words = new Set(rec.words.split(' ')));
|
|
1668
|
+
// A raw identifier's token is `r#name`; source text holds both parts.
|
|
1669
|
+
return words.has(name) || words.has(String(name).replace(/^r#/, '')) || words.has(`r#${name}`);
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
function tokenText(tokens) {
|
|
1673
|
+
return flatten(tokens).map(leaf => leaf.v).join(' ');
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
// ── Build pass ────────────────────────────────────────────────────────────
|
|
1677
|
+
|
|
1678
|
+
/** Project macro names a file's call records invoke. */
|
|
1679
|
+
function invokedMacroNames(index, filePath, inventory) {
|
|
1680
|
+
const calls = index.callsCache.get(filePath)?.calls;
|
|
1681
|
+
if (!Array.isArray(calls)) return [];
|
|
1682
|
+
const names = new Set();
|
|
1683
|
+
for (const call of calls) {
|
|
1684
|
+
if (call.isMacro && !call.macroExpansion && inventory.has(call.name)) names.add(call.name);
|
|
1685
|
+
}
|
|
1686
|
+
return [...names].sort(codeUnitCompare);
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
function signatureFor(state, filePath, names) {
|
|
1690
|
+
const index = state.index;
|
|
1691
|
+
const hash = crypto.createHash('md5');
|
|
1692
|
+
hash.update(index.files.get(filePath)?.hash || '');
|
|
1693
|
+
for (const name of names) {
|
|
1694
|
+
for (const def of state.inventory.get(name) || []) {
|
|
1695
|
+
const entry = index.files.get(def.file);
|
|
1696
|
+
hash.update(`\n${name}\0${def.relativePath}\0${def.startLine}\0${def.endLine}\0${entry?.hash || ''}`);
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
// Module ancestry decides textual scope; crate roots decide path scope.
|
|
1700
|
+
const parents = moduleParents(state);
|
|
1701
|
+
let current = filePath;
|
|
1702
|
+
const seen = new Set();
|
|
1703
|
+
while (parents.has(current) && !seen.has(current)) {
|
|
1704
|
+
seen.add(current);
|
|
1705
|
+
const link = parents.get(current);
|
|
1706
|
+
current = link.file;
|
|
1707
|
+
hash.update(`\n^${path.relative(index.root, current)}\0${link.line}\0${link.macroUse ? 1 : 0}\0` +
|
|
1708
|
+
`${index.files.get(current)?.hash || ''}`);
|
|
1709
|
+
}
|
|
1710
|
+
return hash.digest('hex');
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
/**
|
|
1714
|
+
* Build-time pass, phase 1 (before the import graph): Rust files whose
|
|
1715
|
+
* persisted expansion is missing or stale. An expanded file is re-parsed
|
|
1716
|
+
* (`reindex`) so phase 2 starts from its parser facts.
|
|
1717
|
+
*/
|
|
1718
|
+
function planRustMacroExpansion(index, { reindex = null, files = null } = {}) {
|
|
1719
|
+
const rustFiles = [];
|
|
1720
|
+
for (const [file, entry] of index.files) {
|
|
1721
|
+
if (isRust(entry) && (!files || files.includes(file))) rustFiles.push(file);
|
|
1722
|
+
}
|
|
1723
|
+
if (rustFiles.length === 0) return [];
|
|
1724
|
+
const state = createState(index);
|
|
1725
|
+
const pending = [];
|
|
1726
|
+
for (const file of rustFiles) {
|
|
1727
|
+
const entry = index.files.get(file);
|
|
1728
|
+
const rec = entry.rustMacroExpansion;
|
|
1729
|
+
const names = state.inventory.size > 0 ? invokedMacroNames(index, file, state.inventory) : [];
|
|
1730
|
+
if (names.length === 0 && !rec) continue;
|
|
1731
|
+
if (rec && names.length > 0 && names.join(',') === (rec.callNames || []).join(',') &&
|
|
1732
|
+
signatureFor(state, file, mergedNames(names, rec.nested)) === rec.signature) continue;
|
|
1733
|
+
pending.push(file);
|
|
1734
|
+
}
|
|
1735
|
+
for (const file of pending) {
|
|
1736
|
+
const rec = index.files.get(file)?.rustMacroExpansion;
|
|
1737
|
+
if (rec?.expanded && reindex) reindex(file);
|
|
1738
|
+
const entry = index.files.get(file);
|
|
1739
|
+
if (entry) delete entry.rustMacroExpansion;
|
|
1740
|
+
}
|
|
1741
|
+
return pending;
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
function mergedNames(names, nested) {
|
|
1745
|
+
return [...new Set([...names, ...(nested || [])])].sort(codeUnitCompare);
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
/** Planned files that invoke a project macro (from their call records). */
|
|
1749
|
+
function invokingFiles(index, pending, inventory, names) {
|
|
1750
|
+
return pending.filter(file => {
|
|
1751
|
+
if (!index.files.has(file)) return false;
|
|
1752
|
+
const invoked = invokedMacroNames(index, file, inventory);
|
|
1753
|
+
names.set(file, invoked);
|
|
1754
|
+
return invoked.length > 0;
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
/**
|
|
1759
|
+
* Build-time pass, phase 2a (fix #375): start expanding the planned files in
|
|
1760
|
+
* worker threads while the caller builds the import graph. The workers
|
|
1761
|
+
* resolve module ancestry from the `mod` declarations alone (the only import
|
|
1762
|
+
* facts expansion reads), so their records equal an in-place expansion after
|
|
1763
|
+
* the graph. Returns a handle for applyRustMacroExpansion (null when there is
|
|
1764
|
+
* nothing to expand; `results` null when the files expand in place).
|
|
1765
|
+
*/
|
|
1766
|
+
function startRustMacroExpansion(index, pending, options = {}) {
|
|
1767
|
+
if (!pending || pending.length === 0) return null;
|
|
1768
|
+
const inventory = macroInventory(index);
|
|
1769
|
+
if (inventory.size === 0) return { inventory, files: [], results: null };
|
|
1770
|
+
const names = new Map();
|
|
1771
|
+
const files = invokingFiles(index, pending, inventory, names);
|
|
1772
|
+
let bytes = 0;
|
|
1773
|
+
for (const file of files) bytes += index.files.get(file)?.size || 0;
|
|
1774
|
+
const run = files.length >= PARALLEL_MIN_FILES && bytes >= PARALLEL_MIN_BYTES
|
|
1775
|
+
? startWorkers(index, files, { ...options, inventory }) : null;
|
|
1776
|
+
return { inventory, files, names, results: run };
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
/**
|
|
1780
|
+
* Build-time pass, phase 2b (after the import graph resolved `mod`
|
|
1781
|
+
* declarations): apply the expansions of the planned files, expanding in
|
|
1782
|
+
* place what no worker returned. Returns false when nothing expanded, else
|
|
1783
|
+
* { files, names }: the files whose symbols were replaced and the symbol
|
|
1784
|
+
* names involved.
|
|
1785
|
+
*/
|
|
1786
|
+
function applyRustMacroExpansion(index, pending, options = {}, started = undefined) {
|
|
1787
|
+
const run = started === undefined ? startRustMacroExpansion(index, pending, options) : started;
|
|
1788
|
+
if (!run) return false;
|
|
1789
|
+
let results;
|
|
1790
|
+
try {
|
|
1791
|
+
results = run.results ? run.results.collect() : null;
|
|
1792
|
+
} finally {
|
|
1793
|
+
run.results?.dispose();
|
|
1794
|
+
}
|
|
1795
|
+
if (run.files.length === 0) return false;
|
|
1796
|
+
const state = createState(index, run.inventory);
|
|
1797
|
+
// What the expansions changed: the files whose symbols were replaced and
|
|
1798
|
+
// the symbol names involved (the caller re-canonicalizes only those).
|
|
1799
|
+
let touched = null;
|
|
1800
|
+
for (const file of run.files) {
|
|
1801
|
+
const entry = index.files.get(file);
|
|
1802
|
+
// Symbols-mode expansion never changes call records.
|
|
1803
|
+
const callNames = run.names?.get(file) || invokedMacroNames(index, file, state.inventory);
|
|
1804
|
+
let rec = results?.get(file);
|
|
1805
|
+
if (!rec) {
|
|
1806
|
+
rec = expandFile(state, file) || { sites: 0, blind: [], used: new Map(), expanded: false };
|
|
1807
|
+
rec = { ...rec, usedNames: [...rec.used.values()].map(def => def.name) };
|
|
1808
|
+
}
|
|
1809
|
+
if (rec.symbols) {
|
|
1810
|
+
touched ||= { files: new Set(), names: new Set() };
|
|
1811
|
+
touched.files.add(file);
|
|
1812
|
+
for (const symbol of entry.symbols) touched.names.add(symbol.name);
|
|
1813
|
+
applyGeneratedSymbols(index, file, rec.symbols);
|
|
1814
|
+
for (const symbol of entry.symbols) touched.names.add(symbol.name);
|
|
1815
|
+
}
|
|
1816
|
+
if (rec.expanded && !touched) touched = { files: new Set(), names: new Set() };
|
|
1817
|
+
const nested = [...new Set(rec.usedNames || [])]
|
|
1818
|
+
.filter(name => !callNames.includes(name)).sort(codeUnitCompare);
|
|
1819
|
+
entry.rustMacroExpansion = {
|
|
1820
|
+
signature: signatureFor(state, file, mergedNames(callNames, nested)),
|
|
1821
|
+
callNames,
|
|
1822
|
+
...(nested.length > 0 && { nested }),
|
|
1823
|
+
sites: rec.sites,
|
|
1824
|
+
...(rec.blind.length > 0 && { blind: rec.blind.sort((a, b) => a.line - b.line ||
|
|
1825
|
+
codeUnitCompare(a.macro, b.macro) || codeUnitCompare(a.reason, b.reason)) }),
|
|
1826
|
+
...(rec.expanded && { expanded: true }),
|
|
1827
|
+
...(rec.callsPending && { callsPending: true, calleeNames: rec.calleeNames || [], words: rec.words }),
|
|
1828
|
+
};
|
|
1829
|
+
}
|
|
1830
|
+
return touched || false;
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
// Parse workers kept for expansion pay for themselves from a few files of
|
|
1834
|
+
// expansion work; the job is posted while the import graph builds.
|
|
1835
|
+
const PARALLEL_MIN_FILES = 2;
|
|
1836
|
+
const PARALLEL_MIN_BYTES = 32 * 1024;
|
|
1837
|
+
// Parse workers a parallel build keeps for expansion (fix #388). Their parser
|
|
1838
|
+
// code is already optimized, so a few of them finish the largest invoking
|
|
1839
|
+
// files about as fast as more fresh threads would, at a fraction of the CPU
|
|
1840
|
+
// (measured on ripgrep, tokio, rayon, axum, hyper, cargo, clap).
|
|
1841
|
+
const EXPANSION_WORKERS = 3;
|
|
1842
|
+
|
|
1843
|
+
/**
|
|
1844
|
+
* What a worker needs to expand `workload` without the index: the module
|
|
1845
|
+
* tree (every Rust file's `mod` declarations, resolved, and module symbols),
|
|
1846
|
+
* the macro definitions (their text is read from disk), and for the files
|
|
1847
|
+
* to expand their import facts, impl/trait spans and macro invocation
|
|
1848
|
+
* records. Also every name a project callable bears.
|
|
1849
|
+
*/
|
|
1850
|
+
function workerSnapshot(index, workload, inventory) {
|
|
1851
|
+
const expanding = new Set(workload);
|
|
1852
|
+
// Invocations resolve at the invoking file, and nested ones also at the
|
|
1853
|
+
// defining file.
|
|
1854
|
+
const resolving = new Set(workload);
|
|
1855
|
+
for (const defs of inventory.values()) for (const def of defs) resolving.add(def.file);
|
|
1856
|
+
const files = [];
|
|
1857
|
+
const macros = [];
|
|
1858
|
+
for (const [file, entry] of index.files) {
|
|
1859
|
+
if (!isRust(entry)) continue;
|
|
1860
|
+
const full = expanding.has(file);
|
|
1861
|
+
const resolves = resolving.has(file);
|
|
1862
|
+
// Module ancestry reads `mod` declarations (and the module symbols'
|
|
1863
|
+
// attributes); resolution at a file also reads its glob imports and
|
|
1864
|
+
// the `use` bindings that can name a macro; expansion reads the
|
|
1865
|
+
// invoking file's impl/trait spans and project macro invocations.
|
|
1866
|
+
const record = {
|
|
1867
|
+
path: file, relativePath: entry.relativePath, language: 'rust',
|
|
1868
|
+
importDetails: (entry.importDetails || []).filter(detail => detail.type === 'mod' ||
|
|
1869
|
+
(resolves && detail.type === 'use-glob')),
|
|
1870
|
+
symbols: (entry.symbols || []).filter(symbol => symbol.type === 'module' ||
|
|
1871
|
+
(full && (symbol.type === 'impl' || symbol.type === 'trait')))
|
|
1872
|
+
.map(symbol => ({
|
|
1873
|
+
name: symbol.name, type: symbol.type, startLine: symbol.startLine, endLine: symbol.endLine,
|
|
1874
|
+
modifiers: symbol.modifiers || [],
|
|
1875
|
+
})),
|
|
1876
|
+
};
|
|
1877
|
+
if (resolves) {
|
|
1878
|
+
record.importBindings = (entry.importBindings || [])
|
|
1879
|
+
.filter(binding => inventory.has(binding.alias || binding.name));
|
|
1880
|
+
}
|
|
1881
|
+
if (full) {
|
|
1882
|
+
record.calls = (index.callsCache.get(file)?.calls || [])
|
|
1883
|
+
.filter(call => call.isMacro && !call.macroExpansion && inventory.has(call.name))
|
|
1884
|
+
.map(call => ({
|
|
1885
|
+
name: call.name, line: call.line, callStart: call.callStart, callEnd: call.callEnd,
|
|
1886
|
+
isMacro: true,
|
|
1887
|
+
...(call.inMacro && { inMacro: true }),
|
|
1888
|
+
...(call.macroExpr && { macroExpr: true }),
|
|
1889
|
+
...(call.isPathMacro && { isPathMacro: true, receiver: call.receiver }),
|
|
1890
|
+
...(call.enclosingFunction && { enclosingFunction: { name: call.enclosingFunction.name } }),
|
|
1891
|
+
}));
|
|
1892
|
+
}
|
|
1893
|
+
files.push(record);
|
|
1894
|
+
for (const symbol of entry.symbols || []) {
|
|
1895
|
+
if (symbol.type === 'macro') {
|
|
1896
|
+
macros.push({ name: symbol.name, type: 'macro', file, relativePath: symbol.relativePath,
|
|
1897
|
+
startLine: symbol.startLine, endLine: symbol.endLine, modifiers: symbol.modifiers || [],
|
|
1898
|
+
...(symbol.macroScope && { macroScope: symbol.macroScope }) });
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
// Callee-position macros expand where an argument word names a project
|
|
1903
|
+
// callable: the workers see every such name.
|
|
1904
|
+
const callable = [];
|
|
1905
|
+
for (const [name, defs] of index.symbols) {
|
|
1906
|
+
if (defs.some(def => !NON_CALLABLE_DEF_TYPES.has(def.type))) callable.push(name);
|
|
1907
|
+
}
|
|
1908
|
+
return {
|
|
1909
|
+
root: index.root, files, macros, callableNames: callable,
|
|
1910
|
+
// `mod` declarations resolve in the worker, with this build's view
|
|
1911
|
+
// of the workspace manifests.
|
|
1912
|
+
config: { aliases: index.config?.aliases, includePaths: index.config?.includePaths },
|
|
1913
|
+
cargoManifests: Array.isArray(index.cargoManifests) ? index.cargoManifests : null,
|
|
1914
|
+
};
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
/** An index-shaped view of a worker snapshot (what createState reads). */
|
|
1918
|
+
function snapshotIndex(snapshot) {
|
|
1919
|
+
const { rustModuleDeclarationsResolved, seedWorkspaceManifests } = require('./imports');
|
|
1920
|
+
if (snapshot.cargoManifests) seedWorkspaceManifests(snapshot.root, snapshot.cargoManifests);
|
|
1921
|
+
const files = new Map();
|
|
1922
|
+
const callsCache = new Map();
|
|
1923
|
+
const view = { root: snapshot.root, config: snapshot.config || {}, files };
|
|
1924
|
+
for (const file of snapshot.files) {
|
|
1925
|
+
files.set(file.path, file);
|
|
1926
|
+
if (file.calls) callsCache.set(file.path, { calls: file.calls });
|
|
1927
|
+
let resolved = null;
|
|
1928
|
+
Object.defineProperty(file, 'moduleResolved', {
|
|
1929
|
+
get: () => (resolved ||= rustModuleDeclarationsResolved(view, file.path, file)),
|
|
1930
|
+
});
|
|
1931
|
+
}
|
|
1932
|
+
const symbols = new Map();
|
|
1933
|
+
for (const def of snapshot.macros) {
|
|
1934
|
+
if (!symbols.has(def.name)) symbols.set(def.name, []);
|
|
1935
|
+
symbols.get(def.name).push(def);
|
|
1936
|
+
}
|
|
1937
|
+
let callable = null;
|
|
1938
|
+
return {
|
|
1939
|
+
root: snapshot.root, files, callsCache, symbols,
|
|
1940
|
+
callableNames: {
|
|
1941
|
+
has: word => (callable ||= new Set(snapshot.callableNames || [])).has(word),
|
|
1942
|
+
},
|
|
1943
|
+
_readFile: file => fs.readFileSync(file, 'utf-8'),
|
|
1944
|
+
};
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
/**
|
|
1948
|
+
* Worker-side: expand the files of a shared queue (`files`, next index at
|
|
1949
|
+
* `signal[queueIndex]`), posting each record to `port`; `signal[workerIndex]`
|
|
1950
|
+
* is set when this worker is done.
|
|
1951
|
+
*/
|
|
1952
|
+
function runExpansionQueue({ snapshot, files, signal, workerIndex, queueIndex }, port) {
|
|
1953
|
+
const signalArray = new Int32Array(signal);
|
|
1954
|
+
try {
|
|
1955
|
+
const state = createState(snapshotIndex(JSON.parse(Buffer.from(snapshot).toString('utf8'))));
|
|
1956
|
+
for (;;) {
|
|
1957
|
+
const next = Atomics.add(signalArray, queueIndex, 1);
|
|
1958
|
+
if (next >= files.length) break;
|
|
1959
|
+
const file = files[next];
|
|
1960
|
+
let result;
|
|
1961
|
+
try {
|
|
1962
|
+
result = { file, record: expandForWorker(state, file) };
|
|
1963
|
+
} catch (error) {
|
|
1964
|
+
result = { file, error: error.message };
|
|
1965
|
+
}
|
|
1966
|
+
port.postMessage([result]);
|
|
1967
|
+
}
|
|
1968
|
+
} finally {
|
|
1969
|
+
Atomics.store(signalArray, workerIndex, 1);
|
|
1970
|
+
Atomics.notify(signalArray, workerIndex);
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
/** Worker-side: symbols-mode expansion of one file, as plain records. */
|
|
1975
|
+
function expandForWorker(state, filePath) {
|
|
1976
|
+
const rec = expandFile(state, filePath) || { sites: 0, blind: [], used: new Map(), expanded: false };
|
|
1977
|
+
return {
|
|
1978
|
+
sites: rec.sites, blind: rec.blind, expanded: rec.expanded,
|
|
1979
|
+
usedNames: [...rec.used.values()].map(def => def.name),
|
|
1980
|
+
...(rec.symbols && { symbols: rec.symbols }),
|
|
1981
|
+
...(rec.callsPending && { callsPending: true, calleeNames: rec.calleeNames, words: rec.words }),
|
|
1982
|
+
};
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
/**
|
|
1986
|
+
* Start expanding `files` in the parse workers the build kept for it
|
|
1987
|
+
* (largest first from a shared queue). Returns { collect(), dispose() }, or
|
|
1988
|
+
* null without kept workers (a sequential build expands in place, where the
|
|
1989
|
+
* parser is already optimized); collect() blocks until the workers finish
|
|
1990
|
+
* and returns Map<file, record> (files a worker failed on are expanded in
|
|
1991
|
+
* place by the caller).
|
|
1992
|
+
*/
|
|
1993
|
+
function startWorkers(index, files, options = {}) {
|
|
1994
|
+
const pool = options.pool?.available.length > 0 ? options.pool : null;
|
|
1995
|
+
if (!pool) return null;
|
|
1996
|
+
const { receiveMessageOnPort } = require('worker_threads');
|
|
1997
|
+
const count = Math.min(pool.available.length, EXPANSION_WORKERS, Math.ceil(files.length / 2));
|
|
1998
|
+
if (count < 1) return null;
|
|
1999
|
+
const ordered = [...files].sort((a, b) => (index.files.get(b)?.size || 0) - (index.files.get(a)?.size || 0) ||
|
|
2000
|
+
codeUnitCompare(a, b));
|
|
2001
|
+
// One serialization shared by every worker (a structured clone per
|
|
2002
|
+
// worker cost more than the expansion on large workspaces).
|
|
2003
|
+
const encoded = Buffer.from(JSON.stringify(workerSnapshot(index, files, options.inventory)), 'utf8');
|
|
2004
|
+
const snapshot = new SharedArrayBuffer(encoded.length);
|
|
2005
|
+
encoded.copy(Buffer.from(snapshot));
|
|
2006
|
+
const sab = new SharedArrayBuffer(4 * (count + 1));
|
|
2007
|
+
const signal = new Int32Array(sab);
|
|
2008
|
+
const ports = [];
|
|
2009
|
+
for (let i = 0; i < count; i++) {
|
|
2010
|
+
const worker = pool.available[i];
|
|
2011
|
+
ports.push(pool.ports[worker]);
|
|
2012
|
+
pool.post(worker, { snapshot, files: ordered, signal: sab, workerIndex: i, queueIndex: count });
|
|
2013
|
+
}
|
|
2014
|
+
// Kept workers this job does not need end now.
|
|
2015
|
+
for (const worker of pool.available.slice(count)) pool.retire(worker);
|
|
2016
|
+
// The kept workers belong to the build, which ends them.
|
|
2017
|
+
const dispose = () => {};
|
|
2018
|
+
const collect = () => {
|
|
2019
|
+
const deadline = Date.now() + 300000;
|
|
2020
|
+
let timedOut = false;
|
|
2021
|
+
for (let i = 0; i < count; i++) {
|
|
2022
|
+
while (Atomics.load(signal, i) === 0) {
|
|
2023
|
+
const remaining = deadline - Date.now();
|
|
2024
|
+
if (remaining <= 0) { timedOut = true; break; }
|
|
2025
|
+
Atomics.wait(signal, i, 0, Math.min(remaining, 5000));
|
|
2026
|
+
}
|
|
2027
|
+
if (timedOut) break;
|
|
2028
|
+
}
|
|
2029
|
+
const results = new Map();
|
|
2030
|
+
for (const port of ports) {
|
|
2031
|
+
for (let msg = receiveMessageOnPort(port); msg; msg = receiveMessageOnPort(port)) {
|
|
2032
|
+
for (const result of msg.message) {
|
|
2033
|
+
if (!result.error) results.set(result.file, result.record);
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
return results;
|
|
2038
|
+
};
|
|
2039
|
+
return { collect, dispose, usesPool: true, workerCount: count };
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
/**
|
|
2043
|
+
* Call records of a Rust file whose expansions' calls are not derived yet
|
|
2044
|
+
* (the build derives generated declarations; calls follow on first use and
|
|
2045
|
+
* are then persisted). `materialize: false` (callee index upkeep) returns the
|
|
2046
|
+
* parsed records plus name-only entries for every name the expansions can
|
|
2047
|
+
* call, without expanding.
|
|
2048
|
+
*/
|
|
2049
|
+
function rustExpandedCalls(index, filePath, rawCalls, { materialize = true } = {}) {
|
|
2050
|
+
const rec = index.files.get(filePath)?.rustMacroExpansion;
|
|
2051
|
+
if (!rec?.callsPending || !Array.isArray(rawCalls)) return rawCalls;
|
|
2052
|
+
const hints = (rec.calleeNames || []).map(name => ({ name, line: 0, calleeHint: true }));
|
|
2053
|
+
if (!materialize) return hints.length > 0 ? rawCalls.concat(hints) : rawCalls;
|
|
2054
|
+
if (!index._rustMacroState) index._rustMacroState = createState(index);
|
|
2055
|
+
if (index.calleeIndex && hints.length > 0) index._removeFromCalleeIndex(filePath, hints);
|
|
2056
|
+
rec.callsPending = false;
|
|
2057
|
+
try {
|
|
2058
|
+
expandFile(index._rustMacroState, filePath, 'calls');
|
|
2059
|
+
} catch (error) {
|
|
2060
|
+
rec.callsPending = true;
|
|
2061
|
+
throw error;
|
|
2062
|
+
}
|
|
2063
|
+
delete rec.words;
|
|
2064
|
+
index.macroExpansionDirty = true;
|
|
2065
|
+
return index.callsCache.get(filePath)?.calls || rawCalls;
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
/** Derive every pending file's expansion calls (whole-program consumers). */
|
|
2069
|
+
function materializeRustMacroCalls(index) {
|
|
2070
|
+
for (const [filePath, entry] of index.files) {
|
|
2071
|
+
if (!entry.rustMacroExpansion?.callsPending) continue;
|
|
2072
|
+
const cached = index.callsCache.get(filePath);
|
|
2073
|
+
if (cached) rustExpandedCalls(index, filePath, cached.calls);
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
/** Expanded and blind invocation sites, for health and deadcode disclosure. */
|
|
2078
|
+
function rustMacroExpansionSummary(index) {
|
|
2079
|
+
const summary = { sites: 0, files: 0, blind: { count: 0, fileCount: 0, files: [], sample: [], reasons: {} } };
|
|
2080
|
+
const files = [...index.files.values()].filter(entry => entry.rustMacroExpansion)
|
|
2081
|
+
.sort((a, b) => codeUnitCompare(a.relativePath, b.relativePath));
|
|
2082
|
+
for (const entry of files) {
|
|
2083
|
+
const rec = entry.rustMacroExpansion;
|
|
2084
|
+
summary.sites += rec.sites || 0;
|
|
2085
|
+
if (rec.expanded) summary.files++;
|
|
2086
|
+
const blind = rec.blind || [];
|
|
2087
|
+
if (blind.length === 0) continue;
|
|
2088
|
+
summary.blind.count += blind.length;
|
|
2089
|
+
summary.blind.fileCount++;
|
|
2090
|
+
if (summary.blind.files.length < 10) summary.blind.files.push(entry.relativePath);
|
|
2091
|
+
for (const site of blind) {
|
|
2092
|
+
const reason = site.reason.startsWith('nested:') ? 'nested' : site.reason;
|
|
2093
|
+
summary.blind.reasons[reason] = (summary.blind.reasons[reason] || 0) + 1;
|
|
2094
|
+
if (summary.blind.sample.length < BLIND_SAMPLE) {
|
|
2095
|
+
summary.blind.sample.push({ file: entry.relativePath, ...site });
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
return summary;
|
|
2100
|
+
}
|
|
2101
|
+
|
|
2102
|
+
module.exports = {
|
|
2103
|
+
EXPANSION_WORKERS,
|
|
2104
|
+
planRustMacroExpansion,
|
|
2105
|
+
startRustMacroExpansion,
|
|
2106
|
+
// worker-side (core/build-worker.js)
|
|
2107
|
+
runExpansionQueue,
|
|
2108
|
+
applyRustMacroExpansion,
|
|
2109
|
+
rustExpandedCalls,
|
|
2110
|
+
materializeRustMacroCalls,
|
|
2111
|
+
rustMacroExpansionSummary,
|
|
2112
|
+
rustPendingMayBear,
|
|
2113
|
+
};
|