weavatrix 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +168 -63
- package/SECURITY.md +25 -8
- package/package.json +2 -2
- package/skill/SKILL.md +108 -39
- package/src/analysis/architecture-contract.js +343 -0
- package/src/analysis/audit-debt.js +94 -0
- package/src/analysis/change-classification.js +509 -0
- package/src/analysis/cycle-route.js +14 -0
- package/src/analysis/dead-check.js +13 -6
- package/src/analysis/dead-code-review.js +245 -0
- package/src/analysis/dep-check-ecosystems.js +163 -0
- package/src/analysis/dep-check.js +69 -147
- package/src/analysis/dep-rules.js +47 -31
- package/src/analysis/duplicates.compute.js +47 -7
- package/src/analysis/endpoints-java.js +186 -0
- package/src/analysis/endpoints-rust.js +124 -0
- package/src/analysis/endpoints.js +18 -5
- package/src/analysis/findings.js +8 -1
- package/src/analysis/git-history.js +549 -0
- package/src/analysis/git-ref-graph.js +74 -0
- package/src/analysis/graph-analysis.aggregate.js +29 -28
- package/src/analysis/graph-analysis.edges.js +24 -0
- package/src/analysis/graph-analysis.js +1 -1
- package/src/analysis/graph-analysis.summaries.js +4 -1
- package/src/analysis/http-contracts.js +581 -0
- package/src/analysis/internal-audit.collect.js +43 -2
- package/src/analysis/internal-audit.reach.js +52 -1
- package/src/analysis/internal-audit.run.js +66 -13
- package/src/analysis/java-source.js +36 -0
- package/src/analysis/package-reachability.js +39 -0
- package/src/analysis/static-test-reachability.js +133 -0
- package/src/build-graph.js +72 -13
- package/src/graph/build-worker.js +3 -4
- package/src/graph/builder/lang-java.js +177 -14
- package/src/graph/builder/lang-js.js +71 -12
- package/src/graph/builder/lang-rust.js +129 -6
- package/src/graph/community.js +27 -0
- package/src/graph/file-lock.js +69 -0
- package/src/graph/freshness-probe.js +141 -0
- package/src/graph/graph-filter.js +28 -11
- package/src/graph/incremental-refresh.js +232 -0
- package/src/graph/internal-builder.barrels.js +169 -0
- package/src/graph/internal-builder.build.js +111 -16
- package/src/graph/internal-builder.java.js +33 -0
- package/src/graph/internal-builder.langs.js +2 -1
- package/src/graph/internal-builder.resolvers.js +109 -2
- package/src/graph/layout.js +21 -5
- package/src/graph/relations.js +4 -0
- package/src/graph/repo-registry.js +124 -0
- package/src/mcp/catalog.mjs +64 -21
- package/src/mcp/evidence-snapshot.architecture.mjs +80 -0
- package/src/mcp/evidence-snapshot.common.mjs +160 -0
- package/src/mcp/evidence-snapshot.duplicates.mjs +217 -0
- package/src/mcp/evidence-snapshot.health.mjs +95 -0
- package/src/mcp/evidence-snapshot.inventory.mjs +148 -0
- package/src/mcp/evidence-snapshot.mjs +50 -0
- package/src/mcp/evidence-snapshot.package-graph.mjs +249 -0
- package/src/mcp/evidence-snapshot.structure.mjs +81 -0
- package/src/mcp/graph-context.mjs +106 -181
- package/src/mcp/graph-diff.mjs +217 -0
- package/src/mcp/staleness-notice.mjs +20 -0
- package/src/mcp/sync-evidence.mjs +418 -0
- package/src/mcp/sync-payload.mjs +194 -8
- package/src/mcp/tool-result.mjs +51 -0
- package/src/mcp/tools-actions.mjs +114 -33
- package/src/mcp/tools-architecture.mjs +144 -0
- package/src/mcp/tools-company.mjs +273 -0
- package/src/mcp/tools-graph-hubs.mjs +58 -0
- package/src/mcp/tools-graph.mjs +36 -68
- package/src/mcp/tools-health.mjs +354 -20
- package/src/mcp/tools-history.mjs +22 -0
- package/src/mcp/tools-impact-change.mjs +261 -0
- package/src/mcp/tools-impact.mjs +35 -11
- package/src/mcp-server.mjs +100 -17
- package/src/mcp-source-tools.mjs +11 -3
- package/src/path-classification.js +201 -0
- package/src/path-ignore.js +69 -0
- package/src/security/typosquat.js +1 -1
|
@@ -36,6 +36,18 @@ const FRAMEWORK_RUNTIME_PEERS = new Map([
|
|
|
36
36
|
["@cloudflare/vite-plugin", ["vite", "wrangler"]],
|
|
37
37
|
]);
|
|
38
38
|
|
|
39
|
+
// Style preprocessors are compiler inputs rather than JavaScript imports. Their presence is proven by
|
|
40
|
+
// source extensions in the same package scope, so do not report them as unused merely because Vite,
|
|
41
|
+
// webpack, etc. load them internally. Keep this list to exact compiler/package contracts.
|
|
42
|
+
const IMPLICIT_STYLE_COMPILERS = new Map([
|
|
43
|
+
["sass", /\.(?:scss|sass)$/i],
|
|
44
|
+
["sass-embedded", /\.(?:scss|sass)$/i],
|
|
45
|
+
["less", /\.less$/i],
|
|
46
|
+
["stylus", /\.styl(?:us)?$/i],
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
const isStylesheetSpecifier = (spec) => /\.(?:css|scss|sass|less|styl(?:us)?)(?:[?#].*)?$/i.test(String(spec || ""));
|
|
50
|
+
|
|
39
51
|
const escRe = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
40
52
|
// word-ish mention: the name not embedded inside a longer identifier/path segment
|
|
41
53
|
const mentioned = (blob, name) => new RegExp(`(^|[^\\w@.-])${escRe(name)}($|[^\\w.-])`).test(blob);
|
|
@@ -47,7 +59,7 @@ const mentioned = (blob, name) => new RegExp(`(^|[^\\w@.-])${escRe(name)}($|[^\\
|
|
|
47
59
|
// configTexts — Map<fileName, text> of root config files + CI workflows (mention scanning)
|
|
48
60
|
export function computeDepFindings({
|
|
49
61
|
externalImports = [], pkg = {}, workspacePkgNames = new Set(), configTexts = new Map(),
|
|
50
|
-
aliases = [], scope = "", manifest = "package.json",
|
|
62
|
+
aliases = [], scope = "", manifest = "package.json", nonRuntimeRoots = [], sourceFiles = [],
|
|
51
63
|
} = {}) {
|
|
52
64
|
const findings = [];
|
|
53
65
|
const meta = { scope: scope || ".", manifest };
|
|
@@ -68,6 +80,32 @@ export function computeDepFindings({
|
|
|
68
80
|
};
|
|
69
81
|
const allDeclared = new Set(Object.values(sections).flatMap((s) => Object.keys(s)));
|
|
70
82
|
const selfName = String(pkg.name || "");
|
|
83
|
+
const normRoots = (nonRuntimeRoots || []).map((root) => String(root || "").replace(/\\/g, "/").replace(/^\.\//, "").replace(/^\/+|\/+$/g, "")).filter(Boolean);
|
|
84
|
+
const isNonRuntimeFile = (file) => {
|
|
85
|
+
const normalized = String(file || "").replace(/\\/g, "/").replace(/^\.\//, "");
|
|
86
|
+
return normRoots.some((root) => normalized === root || normalized.startsWith(`${root}/`));
|
|
87
|
+
};
|
|
88
|
+
const npmNameParts = (name) => {
|
|
89
|
+
const value = String(name || "");
|
|
90
|
+
const slash = value.startsWith("@") ? value.indexOf("/") : -1;
|
|
91
|
+
return slash > 0 ? { scope: value.slice(0, slash), base: value.slice(slash + 1) } : { scope: "", base: value };
|
|
92
|
+
};
|
|
93
|
+
const selfParts = npmNameParts(selfName);
|
|
94
|
+
const configuredNapi = npmNameParts(pkg.napi?.name || selfName);
|
|
95
|
+
const napiBinding = {
|
|
96
|
+
scope: configuredNapi.scope || (configuredNapi.base === selfParts.base ? selfParts.scope : ""),
|
|
97
|
+
base: configuredNapi.base,
|
|
98
|
+
};
|
|
99
|
+
const NAPI_PLATFORM_SUFFIX = /^(?:android-(?:arm64|arm-eabi)|win32-(?:x64|ia32|arm64)-msvc|darwin-(?:universal|x64|arm64)|freebsd-(?:x64|arm64)|linux-(?:x64|arm64|arm|riscv64|s390x|ppc64|loong64)-(?:gnu|musl|gnueabihf|musleabihf)|aix-ppc64|sunos-x64|wasm32-wasi(?:-preview1-threads)?)$/;
|
|
100
|
+
const isGeneratedNapiPackage = (name, files) => {
|
|
101
|
+
if (!pkg.napi || !napiBinding.base || !files.every((file) => /(^|\/)index\.[cm]?js$/i.test(String(file || "").replace(/\\/g, "/")))) return false;
|
|
102
|
+
const candidate = npmNameParts(name);
|
|
103
|
+
if (candidate.scope !== napiBinding.scope || !candidate.base.startsWith(`${napiBinding.base}-`)) return false;
|
|
104
|
+
return NAPI_PLATFORM_SUFFIX.test(candidate.base.slice(napiBinding.base.length + 1));
|
|
105
|
+
};
|
|
106
|
+
const isGeneratedNapiBinary = (entry) => !!pkg.napi
|
|
107
|
+
&& /(^|\/)index\.[cm]?js$/i.test(String(entry.file || "").replace(/\\/g, "/"))
|
|
108
|
+
&& /^\.\/[^/]+\.node$/i.test(String(entry.spec || ""));
|
|
71
109
|
// Some frameworks consume required peers internally, so application source legitimately never imports
|
|
72
110
|
// them. Keep this deliberately narrow: react-dom is a required Next.js runtime peer in the same package
|
|
73
111
|
// scope, not a general React exemption and not a repo-wide whitelist.
|
|
@@ -75,9 +113,15 @@ export function computeDepFindings({
|
|
|
75
113
|
for (const [provider, peers] of FRAMEWORK_RUNTIME_PEERS) {
|
|
76
114
|
if (allDeclared.has(provider)) for (const peer of peers) frameworkRuntime.add(peer);
|
|
77
115
|
}
|
|
116
|
+
const implicitCompilerUsage = new Set();
|
|
117
|
+
for (const [name, sourcePattern] of IMPLICIT_STYLE_COMPILERS) {
|
|
118
|
+
if (allDeclared.has(name) && sourceFiles.some((file) => sourcePattern.test(String(file || "")))) {
|
|
119
|
+
implicitCompilerUsage.add(name);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
78
122
|
|
|
79
123
|
// ---- usage index from the graph's recorded imports ----
|
|
80
|
-
const usedPackages = new Map(); // pkgName -> { files:Set, lines:Map(file→first line), kinds:Set }
|
|
124
|
+
const usedPackages = new Map(); // pkgName -> { files:Set, lines:Map(file→first line), kinds/specs:Set, typeOnly:boolean }
|
|
81
125
|
let builtinUsed = false;
|
|
82
126
|
for (const e of externalImports) {
|
|
83
127
|
if (e.ecosystem && e.ecosystem !== "npm") continue; // go/python imports have their own checkers
|
|
@@ -86,10 +130,12 @@ export function computeDepFindings({
|
|
|
86
130
|
if (e.builtin) { builtinUsed = true; continue; }
|
|
87
131
|
if (!e.pkg) continue;
|
|
88
132
|
let u = usedPackages.get(e.pkg);
|
|
89
|
-
if (!u) usedPackages.set(e.pkg, (u = { files: new Set(), lines: new Map(), kinds: new Set() }));
|
|
133
|
+
if (!u) usedPackages.set(e.pkg, (u = { files: new Set(), lines: new Map(), kinds: new Set(), specs: new Set(), typeOnly: true }));
|
|
90
134
|
u.files.add(e.file);
|
|
91
135
|
if (!u.lines.has(e.file)) u.lines.set(e.file, e.line || 0);
|
|
92
136
|
u.kinds.add(e.kind);
|
|
137
|
+
u.specs.add(e.spec || e.pkg);
|
|
138
|
+
if (!e.typeOnly) u.typeOnly = false;
|
|
93
139
|
}
|
|
94
140
|
|
|
95
141
|
const scriptBlob = Object.values(pkg.scripts || {}).join("\n");
|
|
@@ -106,7 +152,7 @@ export function computeDepFindings({
|
|
|
106
152
|
for (const [section, deps] of Object.entries(sections)) {
|
|
107
153
|
if (section === "peerDependencies") continue; // peers are consumer-facing contracts, not usage
|
|
108
154
|
for (const name of Object.keys(deps)) {
|
|
109
|
-
if (name === selfName || usedPackages.has(name) || frameworkRuntime.has(name)) continue;
|
|
155
|
+
if (name === selfName || usedPackages.has(name) || frameworkRuntime.has(name) || implicitCompilerUsage.has(name)) continue;
|
|
110
156
|
const tb = typesBase(name);
|
|
111
157
|
if (tb) { // @types/x is used iff x is used (or it types the Node builtins)
|
|
112
158
|
if (tb === "node" ? builtinUsed : usedPackages.has(tb) || frameworkRuntime.has(tb) || mentioned(configBlob, tb)) continue;
|
|
@@ -121,6 +167,7 @@ export function computeDepFindings({
|
|
|
121
167
|
severity: dev ? "info" : "low",
|
|
122
168
|
confidence: dev || ecosystem ? "low" : "medium",
|
|
123
169
|
title: `Unused ${section === "dependencies" ? "dependency" : section.replace(/ies$/, "y")}: ${name}`,
|
|
170
|
+
reason: "No recorded package import, package-script command, recognized config mention, framework peer contract, or implicit style-compiler input uses this declaration.",
|
|
124
171
|
detail: `"${name}" is declared in ${section} but never imported in source, never referenced by a script, and not mentioned in any known config file. Dynamic/config-convention usage can't be fully ruled out — review before removing.`,
|
|
125
172
|
package: name,
|
|
126
173
|
source: "internal",
|
|
@@ -134,13 +181,21 @@ export function computeDepFindings({
|
|
|
134
181
|
for (const [name, use] of usedPackages) {
|
|
135
182
|
if (allDeclared.has(name) || name === selfName || workspacePkgNames.has(name)) continue;
|
|
136
183
|
const files = [...use.files];
|
|
184
|
+
if (files.every(isNonRuntimeFile) || isGeneratedNapiPackage(name, files)) continue;
|
|
137
185
|
const testOnly = files.every((f) => /(^|[/\\])(test|tests|__tests__|spec|e2e|__mocks__)([/\\]|$)|[._-](test|spec)\.[a-z]+$/i.test(f));
|
|
186
|
+
const stylesheetOnly = [...use.specs].length > 0 && [...use.specs].every(isStylesheetSpecifier);
|
|
187
|
+
const usageReason = stylesheetOnly
|
|
188
|
+
? `Direct stylesheet import(s) resolve through "${name}"; CSS-only imports are build/runtime inputs even without JavaScript bindings.`
|
|
189
|
+
: use.typeOnly
|
|
190
|
+
? `Direct type-only import(s) resolve through "${name}"; the compiler still requires a declared package.`
|
|
191
|
+
: `Direct source import(s) resolve through "${name}", but the nearest package manifest does not declare it.`;
|
|
138
192
|
findings.push(makeFinding({
|
|
139
193
|
category: "unused",
|
|
140
194
|
rule: "missing-dep",
|
|
141
195
|
severity: testOnly ? "low" : "medium",
|
|
142
196
|
confidence: "high",
|
|
143
197
|
title: `Missing dependency: ${name}`,
|
|
198
|
+
reason: usageReason,
|
|
144
199
|
detail: `"${name}" is imported by ${files.length} file(s) in scope ${meta.scope} but not declared in ${manifest} — it only works via a transitive install (phantom dependency) and can break on any lockfile change.`,
|
|
145
200
|
package: name,
|
|
146
201
|
file: files[0],
|
|
@@ -164,6 +219,7 @@ export function computeDepFindings({
|
|
|
164
219
|
severity: "info",
|
|
165
220
|
confidence: "high",
|
|
166
221
|
title: `Duplicate declaration: ${name}`,
|
|
222
|
+
reason: `The same package is declared in multiple manifest sections: ${ss.join(" + ")}.`,
|
|
167
223
|
detail: `"${name}" is declared in ${ss.join(" + ")} — npm resolves one of them; keep a single section.`,
|
|
168
224
|
package: name,
|
|
169
225
|
source: "internal",
|
|
@@ -176,6 +232,7 @@ export function computeDepFindings({
|
|
|
176
232
|
let unresolvedCount = 0;
|
|
177
233
|
for (const e of externalImports) {
|
|
178
234
|
if (!e.unresolved) continue;
|
|
235
|
+
if (isNonRuntimeFile(e.file) || isGeneratedNapiBinary(e)) continue;
|
|
179
236
|
unresolvedCount++;
|
|
180
237
|
const key = e.file + "|" + e.spec;
|
|
181
238
|
if (unresolvedSeen.has(key) || unresolvedSeen.size >= 100) continue; // cap the findings, keep the count honest
|
|
@@ -216,15 +273,21 @@ const ownsFile = (scope, file) => !scope || file === scope || String(file || "")
|
|
|
216
273
|
// Node's package scope and prevents nested Next/Vite apps from inheriting the root manifest by accident.
|
|
217
274
|
export function computeScopedDepFindings({
|
|
218
275
|
externalImports = [], packageScopes = [], workspacePkgNames = new Set(), configTexts = new Map(),
|
|
276
|
+
nonRuntimeRoots = [], sourceFiles = [],
|
|
219
277
|
} = {}) {
|
|
220
278
|
const scopes = packageScopes.length
|
|
221
279
|
? packageScopes.map((s) => ({ ...s, root: normScope(s.root) })).sort((a, b) => b.root.length - a.root.length)
|
|
222
280
|
: [{ root: "", manifest: "package.json", pkg: {}, aliases: [] }];
|
|
223
281
|
const importsByScope = new Map(scopes.map((s) => [s, []]));
|
|
282
|
+
const sourceFilesByScope = new Map(scopes.map((s) => [s, []]));
|
|
224
283
|
for (const e of externalImports) {
|
|
225
284
|
const owner = scopes.find((s) => ownsFile(s.root, e.file)) || scopes[scopes.length - 1];
|
|
226
285
|
importsByScope.get(owner).push(e);
|
|
227
286
|
}
|
|
287
|
+
for (const file of sourceFiles) {
|
|
288
|
+
const owner = scopes.find((s) => ownsFile(s.root, file)) || scopes[scopes.length - 1];
|
|
289
|
+
sourceFilesByScope.get(owner).push(file);
|
|
290
|
+
}
|
|
228
291
|
const configOwner = new Map();
|
|
229
292
|
for (const [file] of configTexts) configOwner.set(file, scopes.find((scope) => ownsFile(scope.root, file)) || scopes[scopes.length - 1]);
|
|
230
293
|
const findings = [], usedPackages = new Map(), declared = new Set();
|
|
@@ -233,6 +296,7 @@ export function computeScopedDepFindings({
|
|
|
233
296
|
const r = computeDepFindings({
|
|
234
297
|
externalImports: importsByScope.get(s), pkg: s.pkg || {}, workspacePkgNames, configTexts: scopeConfig,
|
|
235
298
|
aliases: s.aliases || [], scope: s.root, manifest: s.manifest || (s.root ? `${s.root}/package.json` : "package.json"),
|
|
299
|
+
nonRuntimeRoots, sourceFiles: sourceFilesByScope.get(s),
|
|
236
300
|
});
|
|
237
301
|
findings.push(...r.findings);
|
|
238
302
|
for (const [name, use] of r.usedPackages) usedPackages.set(`${s.root || "."}:${name}`, use);
|
|
@@ -241,146 +305,4 @@ export function computeScopedDepFindings({
|
|
|
241
305
|
return { findings, usedPackages, declared };
|
|
242
306
|
}
|
|
243
307
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
// ---- Go: set math over ecosystem:"Go" externalImports vs go.mod requires ----
|
|
247
|
-
// goMod = parseGoMod() output. Only DIRECT requires can be "unused" (indirect ones belong to `go mod
|
|
248
|
-
// tidy`); replace targets and the own module never count. Missing = imported module with no require
|
|
249
|
-
// prefix — rare in Go (builds fail), so it usually flags vendored/replaced setups: keep it medium.
|
|
250
|
-
export function computeGoDepFindings({ externalImports = [], goMod = null } = {}) {
|
|
251
|
-
const findings = [];
|
|
252
|
-
if (!goMod || !goMod.module) return { findings, declared: new Set() };
|
|
253
|
-
const requires = goMod.requires || [];
|
|
254
|
-
const declared = new Set(requires.map((r) => r.path));
|
|
255
|
-
const replaced = new Set((goMod.replaces || []).map((r) => r.from));
|
|
256
|
-
const moduleOf = (spec) => { let best = ""; for (const p of declared) if ((spec === p || spec.startsWith(p + "/")) && p.length > best.length) best = p; return best; };
|
|
257
|
-
|
|
258
|
-
const usedModules = new Set();
|
|
259
|
-
const missing = new Map(); // pkg → { files:Set, lines:Map }
|
|
260
|
-
for (const e of externalImports) {
|
|
261
|
-
if (e.ecosystem !== "Go" || e.builtin || e.unresolved || !e.pkg) continue;
|
|
262
|
-
const mod = moduleOf(e.spec || e.pkg) || (declared.has(e.pkg) ? e.pkg : "");
|
|
263
|
-
if (mod) { usedModules.add(mod); continue; }
|
|
264
|
-
let m = missing.get(e.pkg);
|
|
265
|
-
if (!m) missing.set(e.pkg, (m = { files: new Set(), lines: new Map() }));
|
|
266
|
-
m.files.add(e.file);
|
|
267
|
-
if (!m.lines.has(e.file)) m.lines.set(e.file, e.line || 0);
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
for (const r of requires) {
|
|
271
|
-
if (r.indirect || usedModules.has(r.path)) continue;
|
|
272
|
-
findings.push(makeFinding({
|
|
273
|
-
category: "unused",
|
|
274
|
-
rule: "unused-dep",
|
|
275
|
-
severity: "low",
|
|
276
|
-
confidence: "medium",
|
|
277
|
-
title: `Unused Go module: ${r.path}`,
|
|
278
|
-
detail: `"${r.path}" is required (direct) in go.mod but no .go file imports it or any of its packages. Build-tag-guarded files can hide usage — confirm with \`go mod tidy\` before removing.`,
|
|
279
|
-
package: r.path,
|
|
280
|
-
version: r.version,
|
|
281
|
-
source: "internal",
|
|
282
|
-
fixHint: "go mod tidy (drops requires nothing imports)",
|
|
283
|
-
}));
|
|
284
|
-
}
|
|
285
|
-
for (const [pkg, use] of missing) {
|
|
286
|
-
if (replaced.has(pkg)) continue;
|
|
287
|
-
const files = [...use.files];
|
|
288
|
-
findings.push(makeFinding({
|
|
289
|
-
category: "unused",
|
|
290
|
-
rule: "missing-dep",
|
|
291
|
-
severity: "medium",
|
|
292
|
-
confidence: "medium",
|
|
293
|
-
title: `Missing Go module: ${pkg}`,
|
|
294
|
-
detail: `"${pkg}" is imported by ${files.length} file(s) but go.mod has no matching require — a replace/workspace/vendor setup, or the module was never added.`,
|
|
295
|
-
package: pkg,
|
|
296
|
-
file: files[0],
|
|
297
|
-
line: use.lines.get(files[0]) || 0,
|
|
298
|
-
evidence: files.slice(0, 5).map((f) => ({ file: f, line: use.lines.get(f) || 0, snippet: "" })),
|
|
299
|
-
source: "internal",
|
|
300
|
-
fixHint: `go get ${pkg}`,
|
|
301
|
-
}));
|
|
302
|
-
}
|
|
303
|
-
return { findings, declared };
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
// ---- Python: ecosystem:"PyPI" externalImports vs requirements/pyproject/Pipfile ----
|
|
307
|
-
// Import→dist naming is heuristic (yaml→PyYAML, python-X/X-python variants), so matching is GENEROUS for
|
|
308
|
-
// suppression and every unused finding stays low-confidence. CLI-only tools (pytest, black, gunicorn …)
|
|
309
|
-
// and stub/plugin conventions (types-*, *-stubs, pytest-*, flake8-*) are never flagged unused.
|
|
310
|
-
const PY_TOOL_DISTS = new Set(("pytest tox nox black ruff flake8 pylint mypy pyright isort bandit coverage pre-commit pip setuptools wheel build twine poetry poetry-core " +
|
|
311
|
-
"pip-tools uv virtualenv pipenv gunicorn uwsgi supervisor ipython jupyter jupyterlab notebook ipykernel codecov autopep8 yapf commitizen detect-secrets safety pip-audit hatchling flit flit-core pdm").split(" "));
|
|
312
|
-
const pyNorm = (n) => String(n || "").toLowerCase().replace(/[-_.]+/g, "-");
|
|
313
|
-
|
|
314
|
-
export function computePyDepFindings({
|
|
315
|
-
externalImports = [], pyManifest = null, configTexts = new Map(),
|
|
316
|
-
managedDependencies = [], ignoredDependencies = [],
|
|
317
|
-
} = {}) {
|
|
318
|
-
const findings = [];
|
|
319
|
-
const deps = (pyManifest && pyManifest.deps) || [];
|
|
320
|
-
const present = !!(pyManifest && pyManifest.present);
|
|
321
|
-
const declared = new Set(deps.map((d) => pyNorm(d.name)));
|
|
322
|
-
const managed = new Set((managedDependencies || []).map(pyNorm));
|
|
323
|
-
const ignored = new Set((ignoredDependencies || []).map(pyNorm));
|
|
324
|
-
for (const name of managed) declared.add(name);
|
|
325
|
-
|
|
326
|
-
const used = new Map(); // top import → { dist, files:Set, lines:Map }
|
|
327
|
-
for (const e of externalImports) {
|
|
328
|
-
if (e.ecosystem !== "PyPI" || e.builtin || e.unresolved || !e.pkg) continue;
|
|
329
|
-
const top = String(e.spec || e.pkg).split(".")[0];
|
|
330
|
-
let u = used.get(top);
|
|
331
|
-
if (!u) used.set(top, (u = { dist: e.pkg, files: new Set(), lines: new Map() }));
|
|
332
|
-
u.files.add(e.file);
|
|
333
|
-
if (!u.lines.has(e.file)) u.lines.set(e.file, e.line || 0);
|
|
334
|
-
}
|
|
335
|
-
// generous match: does declared dist D cover import top t (dist guess g)?
|
|
336
|
-
const covers = (D, t, g) => {
|
|
337
|
-
const d = pyNorm(D), nt = pyNorm(t), ng = pyNorm(g);
|
|
338
|
-
return d === nt || d === ng || d === `python-${nt}` || d === `${nt}-python` || d === `${nt}-binary` || d.replace(/\d+$/, "") === nt.replace(/\d+$/, "");
|
|
339
|
-
};
|
|
340
|
-
|
|
341
|
-
const configBlob = [...configTexts.values()].join("\n");
|
|
342
|
-
if (present) {
|
|
343
|
-
for (const d of deps) {
|
|
344
|
-
const n = pyNorm(d.name);
|
|
345
|
-
if (managed.has(n) || ignored.has(n)) continue;
|
|
346
|
-
if (d.buildSystem || PY_TOOL_DISTS.has(n) || /^types-|-stubs$|^pytest-|^flake8-|^sphinx/.test(n)) continue;
|
|
347
|
-
let hit = false;
|
|
348
|
-
for (const [top, u] of used) if (covers(d.name, top, u.dist)) { hit = true; break; }
|
|
349
|
-
if (hit || mentioned(configBlob, d.name)) continue;
|
|
350
|
-
findings.push(makeFinding({
|
|
351
|
-
category: "unused",
|
|
352
|
-
rule: "unused-dep",
|
|
353
|
-
severity: d.dev ? "info" : "low",
|
|
354
|
-
confidence: "low",
|
|
355
|
-
title: `Unused Python dependency: ${d.name}`,
|
|
356
|
-
detail: `"${d.name}" is declared but no .py file imports a module that maps to it. Import-name↔package-name mapping is heuristic and plugins/CLI tools load dynamically — review before removing.`,
|
|
357
|
-
package: d.name,
|
|
358
|
-
source: "internal",
|
|
359
|
-
fixHint: `remove "${d.name}" from the manifest after confirming nothing imports or shells out to it`,
|
|
360
|
-
}));
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
for (const [top, u] of used) {
|
|
364
|
-
if (ignored.has(pyNorm(top)) || ignored.has(pyNorm(u.dist)) || managed.has(pyNorm(top)) || managed.has(pyNorm(u.dist))) continue;
|
|
365
|
-
let hit = false;
|
|
366
|
-
for (const D of declared) if (covers(D, top, u.dist)) { hit = true; break; }
|
|
367
|
-
if (hit) continue;
|
|
368
|
-
const files = [...u.files];
|
|
369
|
-
const testOnly = files.every((f) => TEST_PATH_RE.test(f));
|
|
370
|
-
findings.push(makeFinding({
|
|
371
|
-
category: "unused",
|
|
372
|
-
rule: "missing-dep",
|
|
373
|
-
severity: present ? (testOnly ? "low" : "medium") : "low",
|
|
374
|
-
confidence: present ? "medium" : "low",
|
|
375
|
-
title: `Missing Python dependency: ${u.dist}`,
|
|
376
|
-
detail: `"${top}" is imported by ${files.length} file(s) but ${present ? "no declared dependency provides it" : "the repo has no Python dependency manifest (requirements.txt / pyproject / Pipfile); a bundled or managed runtime may provide it"}${u.dist !== top ? ` (PyPI package is likely "${u.dist}")` : ""}.${present ? "" : " If this is intentional, declare it under python.managedDependencies in .weavatrix-deps.json."}`,
|
|
377
|
-
package: u.dist,
|
|
378
|
-
file: files[0],
|
|
379
|
-
line: u.lines.get(files[0]) || 0,
|
|
380
|
-
evidence: files.slice(0, 5).map((f) => ({ file: f, line: u.lines.get(f) || 0, snippet: "" })),
|
|
381
|
-
source: "internal",
|
|
382
|
-
fixHint: present ? `pip install ${u.dist} (and add it to requirements.txt / pyproject)` : `add ${u.dist} to a Python manifest, or declare it as a managed runtime dependency`,
|
|
383
|
-
}));
|
|
384
|
-
}
|
|
385
|
-
return { findings, declared, managed };
|
|
386
|
-
}
|
|
308
|
+
export { computeGoDepFindings, computePyDepFindings } from "./dep-check-ecosystems.js";
|
|
@@ -7,26 +7,26 @@
|
|
|
7
7
|
// built from EXTRACTED import edges → high confidence.
|
|
8
8
|
import { makeFinding } from "./findings.js";
|
|
9
9
|
import { ENTRY_FILE } from "./dead-check.js";
|
|
10
|
-
|
|
10
|
+
import { formatRepresentativeCycle } from "./cycle-route.js";
|
|
11
11
|
const TEST_FILE_RE = /(^|[/])(test|tests|__tests__|spec|e2e|__mocks__)([/]|$)|[._-](test|spec)\.[a-z0-9]+$/i;
|
|
12
12
|
// config/data/docs: never "orphans" — nothing imports them by design
|
|
13
13
|
const NON_CODE_RE = /\.(json|ya?ml|sh|ps1|md|txt|html?|css|scss|less)$|(^|[/])(dockerfile|containerfile)/i;
|
|
14
|
-
|
|
15
14
|
const ep = (v) => String(v && typeof v === "object" ? v.id : v);
|
|
16
15
|
const fileOf = (v) => { const s = ep(v); const h = s.indexOf("#"); return h < 0 ? s : s.slice(0, h); };
|
|
17
16
|
const dirOf = (f) => (f.includes("/") ? f.slice(0, f.lastIndexOf("/")) : "");
|
|
18
|
-
|
|
19
17
|
// File-level import adjacency. Go same-directory edges are excluded: a Go package IS the directory, the
|
|
20
18
|
// compiler forbids real cross-package cycles, and our suffix-fallback Go resolution could fake them.
|
|
21
|
-
export function buildFileImportGraph(graph, { includeTypeOnly = false } = {}) {
|
|
19
|
+
export function buildFileImportGraph(graph, { includeTypeOnly = false, includeCompileOnly = false } = {}) {
|
|
22
20
|
const fileIds = new Set();
|
|
23
21
|
for (const n of graph.nodes || []) if (!String(n.id).includes("#")) fileIds.add(String(n.id));
|
|
24
22
|
const runtimeAdj = new Map(); // runtime/value imports only
|
|
25
|
-
const allAdj = new Map(); // runtime +
|
|
23
|
+
const allAdj = new Map(); // runtime + compile-time-only, used to describe architectural coupling
|
|
26
24
|
const edges = [];
|
|
27
25
|
const allEdges = [];
|
|
28
26
|
const typeOnlyEdges = [];
|
|
29
|
-
const
|
|
27
|
+
const compileOnlyEdges = [];
|
|
28
|
+
const compileTimeEdges = [];
|
|
29
|
+
const runtimeSeen = new Set(), allSeen = new Set(), typeSeen = new Set(), compileSeen = new Set(), compileTimeSeen = new Set();
|
|
30
30
|
const add = (map, a, b) => {
|
|
31
31
|
let set = map.get(a);
|
|
32
32
|
if (!set) map.set(a, (set = new Set()));
|
|
@@ -39,25 +39,30 @@ export function buildFileImportGraph(graph, { includeTypeOnly = false } = {}) {
|
|
|
39
39
|
if (a.endsWith(".go") && b.endsWith(".go") && dirOf(a) === dirOf(b)) continue;
|
|
40
40
|
const key = `${a}\0${b}`;
|
|
41
41
|
if (!allSeen.has(key)) { allSeen.add(key); add(allAdj, a, b); allEdges.push([a, b]); }
|
|
42
|
-
if (l.typeOnly === true) {
|
|
43
|
-
if (!typeSeen.has(key)) { typeSeen.add(key); typeOnlyEdges.push([a, b]); }
|
|
42
|
+
if (l.typeOnly === true || l.compileOnly === true) {
|
|
43
|
+
if (l.typeOnly === true && !typeSeen.has(key)) { typeSeen.add(key); typeOnlyEdges.push([a, b]); }
|
|
44
|
+
if (l.compileOnly === true && !compileSeen.has(key)) { compileSeen.add(key); compileOnlyEdges.push([a, b]); }
|
|
45
|
+
if (!compileTimeSeen.has(key)) { compileTimeSeen.add(key); compileTimeEdges.push([a, b]); }
|
|
44
46
|
continue;
|
|
45
47
|
}
|
|
46
48
|
if (!runtimeSeen.has(key)) { runtimeSeen.add(key); add(runtimeAdj, a, b); edges.push([a, b]); }
|
|
47
49
|
}
|
|
48
50
|
const pureTypeOnlyEdges = typeOnlyEdges.filter(([a, b]) => !runtimeSeen.has(`${a}\0${b}`));
|
|
51
|
+
const pureCompileOnlyEdges = compileOnlyEdges.filter(([a, b]) => !runtimeSeen.has(`${a}\0${b}`));
|
|
52
|
+
const pureCompileTimeEdges = compileTimeEdges.filter(([a, b]) => !runtimeSeen.has(`${a}\0${b}`));
|
|
49
53
|
return {
|
|
50
54
|
fileIds,
|
|
51
|
-
adj: includeTypeOnly ? allAdj : runtimeAdj,
|
|
52
|
-
edges: includeTypeOnly ? allEdges : edges,
|
|
55
|
+
adj: includeTypeOnly || includeCompileOnly ? allAdj : runtimeAdj,
|
|
56
|
+
edges: includeTypeOnly || includeCompileOnly ? allEdges : edges,
|
|
53
57
|
runtimeAdj,
|
|
54
58
|
runtimeEdges: edges,
|
|
55
59
|
allAdj,
|
|
56
60
|
allEdges,
|
|
57
61
|
typeOnlyEdges: pureTypeOnlyEdges,
|
|
62
|
+
compileOnlyEdges: pureCompileOnlyEdges,
|
|
63
|
+
compileTimeEdges: pureCompileTimeEdges,
|
|
58
64
|
};
|
|
59
65
|
}
|
|
60
|
-
|
|
61
66
|
// Iterative Tarjan (deep graphs must not blow the JS stack). Returns only non-trivial SCCs (size > 1).
|
|
62
67
|
export function findSccs(adj) {
|
|
63
68
|
const index = new Map(), low = new Map(), onStack = new Set(), S = [];
|
|
@@ -93,7 +98,6 @@ export function findSccs(adj) {
|
|
|
93
98
|
}
|
|
94
99
|
return sccs;
|
|
95
100
|
}
|
|
96
|
-
|
|
97
101
|
// One representative (shortest) cycle through an SCC's lexicographically-first file — a readable path,
|
|
98
102
|
// not the full Johnson enumeration (which explodes combinatorially on big tangles).
|
|
99
103
|
export function representativeCycle(adj, scc) {
|
|
@@ -103,7 +107,7 @@ export function representativeCycle(adj, scc) {
|
|
|
103
107
|
const q = [start];
|
|
104
108
|
while (q.length) {
|
|
105
109
|
const cur = q.shift();
|
|
106
|
-
for (const nxt of adj.get(cur) || []) {
|
|
110
|
+
for (const nxt of [...(adj.get(cur) || [])].sort()) {
|
|
107
111
|
if (nxt === start) {
|
|
108
112
|
const path = [];
|
|
109
113
|
for (let c = cur; c != null; c = prev.get(c)) path.push(c);
|
|
@@ -114,9 +118,9 @@ export function representativeCycle(adj, scc) {
|
|
|
114
118
|
q.push(nxt);
|
|
115
119
|
}
|
|
116
120
|
}
|
|
117
|
-
|
|
121
|
+
const fallback = [...scc].sort();
|
|
122
|
+
return [...fallback, fallback[0]]; // unreachable in a true SCC; deterministic safe fallback
|
|
118
123
|
}
|
|
119
|
-
|
|
120
124
|
// Orphans: file nodes with ZERO non-contains graph degree (nothing in, nothing out — imports, calls,
|
|
121
125
|
// references all collapsed to files). Entries/tests/config-data are exempt. A file that DOES import
|
|
122
126
|
// npm packages (externalImports) is a working script, not an island — confidence drops, not the verdict.
|
|
@@ -140,7 +144,6 @@ export function findOrphans(graph, { entrySet = new Set(), externalImportFiles =
|
|
|
140
144
|
}
|
|
141
145
|
return out;
|
|
142
146
|
}
|
|
143
|
-
|
|
144
147
|
// Boundary DSL — the useful subset of depcruise's forbidden rules, as plain JSON globs:
|
|
145
148
|
// { "forbidden": [{ "name", "comment"?, "severity"?, "from": "main/**", "to": "renderer/**" }],
|
|
146
149
|
// "allowedOnly":[{ "name", "comment"?, "severity"?, "from": "src/ui/**", "to": ["src/ui/**", "src/shared/**"] }] }
|
|
@@ -167,22 +170,23 @@ export function checkBoundaries(edges, rules = {}) {
|
|
|
167
170
|
|
|
168
171
|
const MAX_CYCLE_FINDINGS = 50;
|
|
169
172
|
const MAX_BOUNDARY_FINDINGS = 100;
|
|
170
|
-
|
|
171
173
|
// Assemble everything into unified Findings. rules comes from the repo's .weavatrix-deps.json (optional).
|
|
172
174
|
export function computeStructureFindings(graph, { rules = {}, entrySet = new Set(), externalImportFiles = new Set() } = {}) {
|
|
173
|
-
const { adj, edges, allAdj, allEdges, typeOnlyEdges } = buildFileImportGraph(graph);
|
|
175
|
+
const { adj, edges, allAdj, allEdges, typeOnlyEdges, compileOnlyEdges, compileTimeEdges } = buildFileImportGraph(graph);
|
|
174
176
|
const findings = [];
|
|
175
|
-
|
|
176
177
|
const sccs = findSccs(adj).sort((a, b) => b.length - a.length);
|
|
177
178
|
for (const scc of sccs.slice(0, MAX_CYCLE_FINDINGS)) {
|
|
178
179
|
const cycle = representativeCycle(adj, scc);
|
|
180
|
+
const cycleRoute = formatRepresentativeCycle(cycle);
|
|
179
181
|
findings.push(makeFinding({
|
|
180
182
|
category: "structure",
|
|
181
183
|
rule: "circular-dep",
|
|
182
184
|
severity: scc.length > 4 ? "high" : "medium",
|
|
183
185
|
confidence: "high",
|
|
184
186
|
title: `Circular dependency: ${scc.length} files`,
|
|
185
|
-
detail: `${
|
|
187
|
+
detail: `${cycleRoute}${scc.length + 1 > cycle.length ? ` (representative loop; the tangle spans ${scc.length} files)` : ""}. Break the cycle by extracting the shared piece or inverting one import.`,
|
|
188
|
+
cycleRoute,
|
|
189
|
+
cycleMembers: [...scc].sort(),
|
|
186
190
|
file: cycle[0],
|
|
187
191
|
graphNodeId: cycle[0],
|
|
188
192
|
evidence: cycle.map((f) => ({ file: f, line: 0, snippet: "" })),
|
|
@@ -191,33 +195,40 @@ export function computeStructureFindings(graph, { rules = {}, entrySet = new Set
|
|
|
191
195
|
}));
|
|
192
196
|
}
|
|
193
197
|
|
|
194
|
-
//
|
|
195
|
-
//
|
|
196
|
-
//
|
|
198
|
+
// TypeScript `import type` and Rust module/use edges are compile-time coupling. They can reveal real
|
|
199
|
+
// architecture, but cannot create an initialization-order/runtime cycle. Report SCCs that require
|
|
200
|
+
// either classification separately so agents do not churn working code for a phantom runtime hazard.
|
|
197
201
|
const runtimeKeys = new Set(sccs.map((s) => [...s].sort().join("\0")));
|
|
198
202
|
const allSccs = findSccs(allAdj).sort((a, b) => b.length - a.length);
|
|
199
|
-
const
|
|
203
|
+
const compileTimeCouplings = allSccs.filter((s) => !runtimeKeys.has([...s].sort().join("\0")));
|
|
200
204
|
const edgeCountIn = (scc, list) => {
|
|
201
205
|
const inside = new Set(scc);
|
|
202
206
|
return list.reduce((n, [a, b]) => n + (inside.has(a) && inside.has(b) ? 1 : 0), 0);
|
|
203
207
|
};
|
|
204
|
-
for (const scc of
|
|
208
|
+
for (const scc of compileTimeCouplings.slice(0, MAX_CYCLE_FINDINGS)) {
|
|
205
209
|
const cycle = representativeCycle(allAdj, scc);
|
|
210
|
+
const cycleRoute = formatRepresentativeCycle(cycle);
|
|
206
211
|
const runtimeInside = edgeCountIn(scc, edges);
|
|
207
212
|
const typeInside = edgeCountIn(scc, typeOnlyEdges);
|
|
213
|
+
const compileInside = edgeCountIn(scc, compileOnlyEdges);
|
|
208
214
|
const containsRuntimeCycle = sccs.some((runtime) => runtime.every((f) => scc.includes(f)));
|
|
215
|
+
const typeSpecific = compileInside === 0;
|
|
209
216
|
findings.push(makeFinding({
|
|
210
217
|
category: "structure",
|
|
211
|
-
rule: "type-coupling",
|
|
218
|
+
rule: typeSpecific ? "type-coupling" : "compile-time-coupling",
|
|
212
219
|
severity: "info",
|
|
213
220
|
confidence: "high",
|
|
214
|
-
title: `${containsRuntimeCycle
|
|
215
|
-
|
|
221
|
+
title: `${containsRuntimeCycle
|
|
222
|
+
? (typeSpecific ? "Type imports expand dependency coupling" : "Compile-time edges expand dependency coupling")
|
|
223
|
+
: (typeSpecific ? "Type-induced dependency cycle (no runtime cycle)" : "Compile-time dependency cycle (no runtime cycle)")}: ${scc.length} files`,
|
|
224
|
+
detail: `${cycleRoute}. This strongly-connected group needs compile-time-only edges to close; it contains ${runtimeInside} runtime edge(s), ${typeInside} type-only edge(s), and ${compileInside} compile-only edge(s)${containsRuntimeCycle ? ", with a smaller runtime cycle reported separately" : ", while its runtime import graph is acyclic"}. Treat this as design coupling, not an initialization-order failure.`,
|
|
225
|
+
cycleRoute,
|
|
226
|
+
cycleMembers: [...scc].sort(),
|
|
216
227
|
file: cycle[0],
|
|
217
228
|
graphNodeId: cycle[0],
|
|
218
229
|
evidence: cycle.map((f) => ({ file: f, line: 0, snippet: "" })),
|
|
219
230
|
source: "internal",
|
|
220
|
-
fixHint: "review the
|
|
231
|
+
fixHint: "review the compile-time ownership only if the coupling impedes changes; no runtime-cycle fix is required",
|
|
221
232
|
}));
|
|
222
233
|
}
|
|
223
234
|
if (sccs.length > MAX_CYCLE_FINDINGS) {
|
|
@@ -275,11 +286,16 @@ export function computeStructureFindings(graph, { rules = {}, entrySet = new Set
|
|
|
275
286
|
importEdges: allEdges.length,
|
|
276
287
|
runtimeImportEdges: edges.length,
|
|
277
288
|
typeOnlyImportEdges: typeOnlyEdges.length,
|
|
289
|
+
compileOnlyImportEdges: compileOnlyEdges.length,
|
|
290
|
+
compileTimeImportEdges: compileTimeEdges.length,
|
|
278
291
|
cycles: sccs.length,
|
|
279
292
|
runtimeCycles: sccs.length,
|
|
280
293
|
largestCycle: sccs[0]?.length || 0,
|
|
281
|
-
|
|
282
|
-
|
|
294
|
+
// Backward-compatible aliases remain for edgeTypesV 1 consumers.
|
|
295
|
+
typeCouplings: compileTimeCouplings.length,
|
|
296
|
+
largestTypeCoupling: compileTimeCouplings[0]?.length || 0,
|
|
297
|
+
compileTimeCouplings: compileTimeCouplings.length,
|
|
298
|
+
largestCompileTimeCoupling: compileTimeCouplings[0]?.length || 0,
|
|
283
299
|
orphans: findings.filter((f) => f.rule === "orphan-file").length,
|
|
284
300
|
boundaryViolations: violations.length,
|
|
285
301
|
},
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// duplicates.compute.js — fragment extraction, inverted-index pairing, and the computeDuplicates
|
|
2
2
|
// pipeline (split from duplicates.js; see the facade there for the full algorithm overview).
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
|
-
import { isTestPath } from "../graph/graph-filter.js";
|
|
5
4
|
import { createRepoBoundary } from "../repo-path.js";
|
|
5
|
+
import { createPathClassifier, hasPathClass } from "../path-classification.js";
|
|
6
6
|
import { listRepoFiles } from "./internal-audit.collect.js";
|
|
7
7
|
import { stripNonCode, bodyEndLineCount, tokenize, fingerprints, extractLargeStrings } from "./duplicates.tokenize.js";
|
|
8
8
|
|
|
@@ -113,9 +113,19 @@ function symbolRanges(graph) {
|
|
|
113
113
|
return byFile;
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
+
function loadGraph(graphInput) {
|
|
117
|
+
if (graphInput && typeof graphInput === "object" && !Array.isArray(graphInput)) {
|
|
118
|
+
return { nodes: Array.isArray(graphInput.nodes) ? graphInput.nodes : [] };
|
|
119
|
+
}
|
|
120
|
+
if (typeof graphInput !== "string" || !graphInput) throw new TypeError("graph path or graph object is required");
|
|
121
|
+
const parsed = JSON.parse(readFileSync(graphInput, "utf8"));
|
|
122
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new TypeError("graph must be an object");
|
|
123
|
+
return { nodes: Array.isArray(parsed.nodes) ? parsed.nodes : [] };
|
|
124
|
+
}
|
|
125
|
+
|
|
116
126
|
export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
117
127
|
const includeStrings = !!opts.includeStrings;
|
|
118
|
-
const graph =
|
|
128
|
+
const graph = loadGraph(graphJsonPath);
|
|
119
129
|
const byFile = symbolRanges(graph);
|
|
120
130
|
// total symbol nodes in the graph — 0 means a file-only graph (built by an older builder before
|
|
121
131
|
// symbol extraction, or a stale graph): clone detection has nothing to work with, and the UI must
|
|
@@ -126,6 +136,18 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
126
136
|
for (const [file, arr] of byFile) if (allowedFiles.has(file)) graphSymbols += arr.length;
|
|
127
137
|
const frags = [];
|
|
128
138
|
const boundary = createRepoBoundary(repoPath);
|
|
139
|
+
const classifier = createPathClassifier(repoPath);
|
|
140
|
+
const classificationByFile = new Map();
|
|
141
|
+
const classify = (file, content) => {
|
|
142
|
+
if (!classificationByFile.has(file)) classificationByFile.set(file, classifier.explain(file, { content }));
|
|
143
|
+
return classificationByFile.get(file);
|
|
144
|
+
};
|
|
145
|
+
const classificationFields = (info) => ({
|
|
146
|
+
test: hasPathClass(info, "test", "e2e"),
|
|
147
|
+
classes: info.classes,
|
|
148
|
+
excluded: info.excluded,
|
|
149
|
+
matchedRule: info.matchedRule,
|
|
150
|
+
});
|
|
129
151
|
for (const [file, syms] of byFile) {
|
|
130
152
|
if (!allowedFiles.has(file)) continue;
|
|
131
153
|
const resolved = boundary.resolve(file);
|
|
@@ -134,6 +156,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
134
156
|
let lines;
|
|
135
157
|
try { lines = readFileSync(full, "utf8").split(/\r?\n/); } catch { continue; }
|
|
136
158
|
const py = /\.py$/i.test(file);
|
|
159
|
+
const pathInfo = classify(file, lines.join("\n"));
|
|
137
160
|
for (let i = 0; i < syms.length; i++) {
|
|
138
161
|
const start = syms[i].line;
|
|
139
162
|
// hard cap first (next symbol, or EOF, whichever comes first, ≤ MAX_BODY_LINES) …
|
|
@@ -150,7 +173,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
150
173
|
frags.push({
|
|
151
174
|
id: syms[i].id, label: syms[i].label, file, start, end,
|
|
152
175
|
n: toks.strict.length,
|
|
153
|
-
|
|
176
|
+
...classificationFields(pathInfo),
|
|
154
177
|
fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
|
|
155
178
|
});
|
|
156
179
|
}
|
|
@@ -164,7 +187,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
164
187
|
if (toks.strict.length < FLOOR_TOKENS) return;
|
|
165
188
|
frags.push({
|
|
166
189
|
id: `${file}#str@${startLine}`, label: `${base}:${startLine} ~${endLine - startLine + 1}-line string`,
|
|
167
|
-
file, start: startLine, end: endLine, n: toks.strict.length,
|
|
190
|
+
file, start: startLine, end: endLine, n: toks.strict.length, ...classificationFields(pathInfo), kind: "string",
|
|
168
191
|
fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
|
|
169
192
|
});
|
|
170
193
|
};
|
|
@@ -188,13 +211,15 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
188
211
|
// ---- windowed fragments for symbol-less file types (CSS/HTML/MD/…), found by walking the repo (NOT the
|
|
189
212
|
// graph — assets are dedup-only). Each file is sliced into WINDOW_LINES blocks and fingerprinted.
|
|
190
213
|
const symFiles = new Set(byFile.keys());
|
|
191
|
-
const
|
|
214
|
+
const allAssetFiles = repoFiles.filter((file) => WINDOW_EXTS.test(file));
|
|
215
|
+
const assetFiles = allAssetFiles.slice(0, MAX_ASSET_FILES);
|
|
192
216
|
for (const file of assetFiles) {
|
|
193
217
|
if (symFiles.has(file)) continue;
|
|
194
218
|
const resolved = boundary.resolve(file);
|
|
195
219
|
if (!resolved.ok) continue;
|
|
196
220
|
let lines;
|
|
197
221
|
try { lines = readFileSync(resolved.path, "utf8").split(/\r?\n/); } catch { continue; }
|
|
222
|
+
const pathInfo = classify(file, lines.join("\n"));
|
|
198
223
|
for (let start = 1; start <= lines.length; start += WINDOW_LINES) {
|
|
199
224
|
const end = Math.min(start + WINDOW_LINES - 1, lines.length);
|
|
200
225
|
if (end - start < 2) continue;
|
|
@@ -202,7 +227,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
202
227
|
if (toks.strict.length < FLOOR_TOKENS) continue;
|
|
203
228
|
frags.push({
|
|
204
229
|
id: `${file}#win@${start}`, label: `${file.split("/").pop()}:${start}-${end}`, file, start, end,
|
|
205
|
-
n: toks.strict.length,
|
|
230
|
+
n: toks.strict.length, ...classificationFields(pathInfo),
|
|
206
231
|
fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
|
|
207
232
|
});
|
|
208
233
|
}
|
|
@@ -211,5 +236,20 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
211
236
|
const nameTwins = opts.nameTwins ? computeNameTwins(frags) : null;
|
|
212
237
|
// fp sets are worker-internal — strip them from the payload that crosses the thread boundary
|
|
213
238
|
const slim = frags.map(({ fp, ...rest }) => rest);
|
|
214
|
-
return {
|
|
239
|
+
return {
|
|
240
|
+
ok: true,
|
|
241
|
+
frags: slim,
|
|
242
|
+
modes,
|
|
243
|
+
...(nameTwins ? { nameTwins } : {}),
|
|
244
|
+
graphSymbols,
|
|
245
|
+
completeness: {
|
|
246
|
+
assetFiles: {
|
|
247
|
+
total: allAssetFiles.length,
|
|
248
|
+
scanned: assetFiles.length,
|
|
249
|
+
truncated: allAssetFiles.length > assetFiles.length,
|
|
250
|
+
},
|
|
251
|
+
nameTwinsTruncated: !!nameTwins && nameTwins.length >= 200,
|
|
252
|
+
},
|
|
253
|
+
floors: { tokens: FLOOR_TOKENS, sim: FLOOR_SIM * 100 },
|
|
254
|
+
};
|
|
215
255
|
}
|