weavatrix 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -53
- package/SECURITY.md +25 -8
- package/package.json +2 -2
- package/skill/SKILL.md +92 -30
- package/src/analysis/architecture-contract.js +343 -0
- package/src/analysis/audit-debt.js +94 -0
- package/src/analysis/change-classification.js +509 -0
- package/src/analysis/cycle-route.js +14 -0
- package/src/analysis/dead-check.js +5 -3
- package/src/analysis/dead-code-review.js +245 -0
- package/src/analysis/dep-check-ecosystems.js +6 -0
- package/src/analysis/dep-check.js +40 -6
- package/src/analysis/dep-rules.js +12 -9
- package/src/analysis/duplicates.compute.js +47 -7
- package/src/analysis/endpoints-java.js +186 -0
- package/src/analysis/endpoints.js +8 -2
- package/src/analysis/findings.js +8 -1
- package/src/analysis/git-history.js +549 -0
- package/src/analysis/git-ref-graph.js +74 -0
- package/src/analysis/graph-analysis.aggregate.js +2 -2
- package/src/analysis/graph-analysis.edges.js +3 -0
- package/src/analysis/graph-analysis.summaries.js +1 -1
- package/src/analysis/http-contracts.js +581 -0
- package/src/analysis/internal-audit.collect.js +3 -2
- package/src/analysis/internal-audit.reach.js +52 -1
- package/src/analysis/internal-audit.run.js +58 -10
- package/src/analysis/java-source.js +36 -0
- package/src/analysis/package-reachability.js +39 -0
- package/src/analysis/static-test-reachability.js +133 -0
- package/src/build-graph.js +72 -13
- package/src/graph/build-worker.js +3 -4
- package/src/graph/builder/lang-js.js +71 -12
- package/src/graph/community.js +10 -0
- package/src/graph/file-lock.js +69 -0
- package/src/graph/freshness-probe.js +141 -0
- package/src/graph/graph-filter.js +28 -11
- package/src/graph/incremental-refresh.js +232 -0
- package/src/graph/internal-builder.barrels.js +169 -0
- package/src/graph/internal-builder.build.js +82 -11
- package/src/graph/internal-builder.langs.js +2 -1
- package/src/graph/layout.js +21 -5
- package/src/graph/repo-registry.js +124 -0
- package/src/mcp/catalog.mjs +62 -19
- package/src/mcp/evidence-snapshot.architecture.mjs +80 -0
- package/src/mcp/evidence-snapshot.common.mjs +160 -0
- package/src/mcp/evidence-snapshot.duplicates.mjs +217 -0
- package/src/mcp/evidence-snapshot.health.mjs +95 -0
- package/src/mcp/evidence-snapshot.inventory.mjs +148 -0
- package/src/mcp/evidence-snapshot.mjs +50 -0
- package/src/mcp/evidence-snapshot.package-graph.mjs +249 -0
- package/src/mcp/evidence-snapshot.structure.mjs +81 -0
- package/src/mcp/graph-context.mjs +100 -16
- package/src/mcp/graph-diff.mjs +74 -20
- package/src/mcp/staleness-notice.mjs +20 -0
- package/src/mcp/sync-evidence.mjs +418 -0
- package/src/mcp/sync-payload.mjs +164 -0
- package/src/mcp/tool-result.mjs +51 -0
- package/src/mcp/tools-actions.mjs +111 -30
- package/src/mcp/tools-architecture.mjs +144 -0
- package/src/mcp/tools-company.mjs +273 -0
- package/src/mcp/tools-graph.mjs +25 -14
- package/src/mcp/tools-health.mjs +336 -14
- package/src/mcp/tools-history.mjs +22 -0
- package/src/mcp/tools-impact-change.mjs +261 -0
- package/src/mcp/tools-impact.mjs +25 -6
- package/src/mcp-server.mjs +100 -17
- package/src/mcp-source-tools.mjs +11 -3
- package/src/path-classification.js +201 -0
- package/src/path-ignore.js +69 -0
- package/src/security/typosquat.js +1 -1
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// Conservative, source-free review queue for statically unreferenced code. This deliberately builds
|
|
2
|
+
// on computeDead instead of inventing a second liveness model. Candidates are evidence for review,
|
|
3
|
+
// never deletion instructions: framework entry, dynamic loading, reflection and public API surfaces
|
|
4
|
+
// lower confidence and remain explicit in the returned record.
|
|
5
|
+
import { computeDead, isFrameworkEntryFile } from "./dead-check.js";
|
|
6
|
+
import { createPathClassifier, hasPathClass } from "../path-classification.js";
|
|
7
|
+
|
|
8
|
+
const CONFIDENCE_RANK = Object.freeze({ high: 0, medium: 1, low: 2 });
|
|
9
|
+
const CLASSIFIED_NON_PRODUCT = Object.freeze(["generated", "mock", "story", "docs", "benchmark", "temp"]);
|
|
10
|
+
const DYNAMIC_RE = /(?:\bimport\s*\(|\brequire\s*\(\s*(?!["'])|\bcreateRequire\s*\(|\b__import__\s*\(|\bimportlib\.|(?:^|[^\w.$])(?:eval|exec)\s*\()/m;
|
|
11
|
+
const REFLECTION_RE = /(?:\b(?:Class\.forName|get(?:Declared)?Method|getattr|setattr|hasattr|Method\.Invoke|GetMethod|GetProcAddress|dlsym)\s*\(|\b(?:globals|locals)\s*\(\s*\)\s*\[|\breflect\.[A-Za-z_$][\w$]*\s*\()/i;
|
|
12
|
+
|
|
13
|
+
const normalizedPath = (value) => String(value || "").replace(/\\/g, "/").replace(/^\.\//, "");
|
|
14
|
+
const lineOf = (node) => {
|
|
15
|
+
const match = /@(\d+)$/.exec(String(node?.id || "")) || /L(\d+)/.exec(String(node?.source_location || ""));
|
|
16
|
+
return match ? Number(match[1]) : 0;
|
|
17
|
+
};
|
|
18
|
+
const bareLabel = (value) => String(value || "").replace(/\s*\(.*$/, "").replace(/[()]/g, "").trim();
|
|
19
|
+
|
|
20
|
+
function kindOf(node) {
|
|
21
|
+
const symbolKind = String(node?.symbol_kind || "").toLowerCase();
|
|
22
|
+
if (symbolKind === "method" || symbolKind === "constructor") return "method";
|
|
23
|
+
if (["function", "function_definition", "func", "fn"].includes(symbolKind)) return "function";
|
|
24
|
+
// Old graphs may not retain symbol_kind. Only use call syntax as a compatibility fallback;
|
|
25
|
+
// member_of alone also describes fields and must never promote them to methods.
|
|
26
|
+
if (!symbolKind && /\([^)]*\)\s*$/.test(String(node?.label || ""))) return node?.member_of ? "method" : "function";
|
|
27
|
+
return "symbol";
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function isPublicSurface(node) {
|
|
31
|
+
const visibility = String(node?.visibility || "").toLowerCase();
|
|
32
|
+
return node?.exported === true || visibility === "public" || visibility === "protected";
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function pathAllowed(info, { includeTests, includeClassified }) {
|
|
36
|
+
if (!includeTests && hasPathClass(info, "test", "e2e")) return { ok: false, bucket: "tests" };
|
|
37
|
+
if (!includeClassified && (info?.excluded || hasPathClass(info, ...CLASSIFIED_NON_PRODUCT))) {
|
|
38
|
+
return { ok: false, bucket: "classified" };
|
|
39
|
+
}
|
|
40
|
+
return { ok: true };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function symbolCandidate(item, node, context) {
|
|
44
|
+
const file = normalizedPath(item.file);
|
|
45
|
+
const source = String(context.sources.get(file) || "");
|
|
46
|
+
const pathInfo = context.classify(file, source);
|
|
47
|
+
const publicSurface = isPublicSurface(node);
|
|
48
|
+
const externalEntry = context.entrySet.has(file) || isFrameworkEntryFile(file);
|
|
49
|
+
const framework = context.frameworkByFile.get(file) || null;
|
|
50
|
+
const dynamicFile = context.dynamicTargets.has(file) || DYNAMIC_RE.test(source);
|
|
51
|
+
const reflectionFile = REFLECTION_RE.test(source);
|
|
52
|
+
const kind = kindOf(node);
|
|
53
|
+
let confidence = (String(node?.visibility || "").toLowerCase() === "private" || (!publicSurface && ["method", "function"].includes(kind)))
|
|
54
|
+
? "high" : "medium";
|
|
55
|
+
const caveats = [];
|
|
56
|
+
|
|
57
|
+
if (publicSurface) {
|
|
58
|
+
confidence = "low";
|
|
59
|
+
caveats.push("Public/exported APIs can be consumed by downstream packages, interfaces, reflection, dependency injection, templates, or configuration outside this repository.");
|
|
60
|
+
}
|
|
61
|
+
if (externalEntry || framework) {
|
|
62
|
+
confidence = "low";
|
|
63
|
+
caveats.push(framework?.reason || "This file is an externally entered or framework-owned surface; static inbound edges are not complete usage evidence.");
|
|
64
|
+
}
|
|
65
|
+
if (node?.decorated) {
|
|
66
|
+
confidence = "low";
|
|
67
|
+
caveats.push("Decorators/annotations can register this symbol without a direct caller.");
|
|
68
|
+
}
|
|
69
|
+
if (dynamicFile) {
|
|
70
|
+
confidence = "low";
|
|
71
|
+
caveats.push("The declaring file uses or is reached through dynamic loading, so the static graph can miss callers.");
|
|
72
|
+
}
|
|
73
|
+
if (reflectionFile || (publicSurface && context.repoSignals.reflection)) {
|
|
74
|
+
confidence = "low";
|
|
75
|
+
caveats.push("Reflection is present and may invoke names without a resolvable static edge.");
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return {
|
|
79
|
+
id: String(item.id),
|
|
80
|
+
kind,
|
|
81
|
+
classification: publicSurface ? `public-${kind}` : kind === "method" ? "internal-method" : kind === "function" ? "internal-function" : "unreferenced-symbol",
|
|
82
|
+
file,
|
|
83
|
+
line: lineOf(node),
|
|
84
|
+
symbol: bareLabel(node?.label || item.label),
|
|
85
|
+
owner: node?.member_of || null,
|
|
86
|
+
symbolKind: node?.symbol_kind || null,
|
|
87
|
+
visibility: node?.visibility || (node?.exported ? "exported" : "internal"),
|
|
88
|
+
confidence,
|
|
89
|
+
reason: item.reason,
|
|
90
|
+
evidence: [
|
|
91
|
+
{ kind: "graph", fact: "No inbound non-structural graph edge targets this symbol." },
|
|
92
|
+
{ kind: "source-index", fact: "Its identifier has no second indexed occurrence that establishes a caller." },
|
|
93
|
+
],
|
|
94
|
+
caveats,
|
|
95
|
+
publicApi: publicSurface,
|
|
96
|
+
externallyEnteredFile: externalEntry,
|
|
97
|
+
pathClasses: pathInfo.classes || [],
|
|
98
|
+
matchedPathRule: pathInfo.matchedRule || null,
|
|
99
|
+
reviewAction: "Confirm with read_source, get_dependents, exact search, framework/config inspection and tests; never auto-delete.",
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function fileCandidate(item, symbols, context) {
|
|
104
|
+
const file = normalizedPath(item.file);
|
|
105
|
+
const source = String(context.sources.get(file) || "");
|
|
106
|
+
const pathInfo = context.classify(file, source);
|
|
107
|
+
const publicSymbols = symbols.filter((symbol) => symbol.publicApi);
|
|
108
|
+
const dynamicFile = context.dynamicTargets.has(file) || DYNAMIC_RE.test(source);
|
|
109
|
+
const reflectionFile = REFLECTION_RE.test(source);
|
|
110
|
+
// Whole-file liveness always remains at most medium: external launchers/manifests can exist outside
|
|
111
|
+
// the indexed import graph even when every internal symbol signal is otherwise strong.
|
|
112
|
+
let confidence = "medium";
|
|
113
|
+
const caveats = ["Files can be launched by scripts, plugins, manifests, framework conventions, generated consumers, or external tooling without an import edge."];
|
|
114
|
+
if (publicSymbols.length) {
|
|
115
|
+
confidence = "low";
|
|
116
|
+
caveats.push(`${publicSymbols.length} indexed public/exported symbol(s) may be consumed outside this repository.`);
|
|
117
|
+
}
|
|
118
|
+
if (dynamicFile) {
|
|
119
|
+
confidence = "low";
|
|
120
|
+
caveats.push("Dynamic loading is present in or targets this file.");
|
|
121
|
+
}
|
|
122
|
+
if (reflectionFile) {
|
|
123
|
+
confidence = "low";
|
|
124
|
+
caveats.push("Reflection is present in this file.");
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
id: `file:${file}`,
|
|
128
|
+
kind: "file",
|
|
129
|
+
classification: "unreferenced-file",
|
|
130
|
+
file,
|
|
131
|
+
line: 1,
|
|
132
|
+
symbol: null,
|
|
133
|
+
owner: null,
|
|
134
|
+
symbolKind: null,
|
|
135
|
+
visibility: null,
|
|
136
|
+
confidence,
|
|
137
|
+
reason: item.reason,
|
|
138
|
+
evidence: [
|
|
139
|
+
{ kind: "graph", fact: "No indexed module imports this file." },
|
|
140
|
+
{ kind: "symbol-liveness", fact: "Every indexed symbol in the file is statically unreferenced." },
|
|
141
|
+
],
|
|
142
|
+
caveats,
|
|
143
|
+
publicApi: publicSymbols.length > 0,
|
|
144
|
+
externallyEnteredFile: false,
|
|
145
|
+
pathClasses: pathInfo.classes || [],
|
|
146
|
+
matchedPathRule: pathInfo.matchedRule || null,
|
|
147
|
+
reviewAction: "Verify package scripts, manifests, framework discovery, dynamic loading and external consumers; never auto-delete.",
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Pure review model. Filesystem collection/entry inference stays in the MCP adapter so tests can pass
|
|
152
|
+
// exact source maps and convention evidence without touching the working tree.
|
|
153
|
+
export function computeDeadCodeReview(graph, sources, options = {}) {
|
|
154
|
+
const entrySet = options.entrySet instanceof Set ? options.entrySet : new Set(options.entrySet || []);
|
|
155
|
+
const dynamicTargets = options.dynamicTargets instanceof Set ? options.dynamicTargets : new Set(options.dynamicTargets || []);
|
|
156
|
+
const frameworkByFile = new Map((options.frameworkEvidence || []).map((entry) => [normalizedPath(entry.file), entry]));
|
|
157
|
+
const classifier = options.pathClassifier || createPathClassifier(null);
|
|
158
|
+
const classificationCache = new Map();
|
|
159
|
+
const classify = (file, source) => {
|
|
160
|
+
if (!classificationCache.has(file)) classificationCache.set(file, classifier.explain(file, { content: source }));
|
|
161
|
+
return classificationCache.get(file);
|
|
162
|
+
};
|
|
163
|
+
const repoSignals = {
|
|
164
|
+
dynamicLoading: (graph.externalImports || []).some((entry) => entry?.dynamic) || [...sources.values()].some((text) => DYNAMIC_RE.test(String(text || ""))),
|
|
165
|
+
reflection: [...sources.values()].some((text) => REFLECTION_RE.test(String(text || ""))),
|
|
166
|
+
};
|
|
167
|
+
const includeTests = options.includeTests === true;
|
|
168
|
+
const includeClassified = options.includeClassified === true;
|
|
169
|
+
const minConfidence = Object.hasOwn(CONFIDENCE_RANK, options.minConfidence) ? options.minConfidence : "medium";
|
|
170
|
+
const pathPrefix = normalizedPath(options.path || "").replace(/\/+$/, "");
|
|
171
|
+
const requestedKinds = new Set(Array.isArray(options.kinds) && options.kinds.length ? options.kinds : ["file", "function", "method", "symbol"]);
|
|
172
|
+
const context = { sources, entrySet, dynamicTargets, frameworkByFile, classify, repoSignals };
|
|
173
|
+
const dead = computeDead(graph, sources, { entrySet });
|
|
174
|
+
const nodesById = new Map((graph.nodes || []).map((node) => [String(node.id), node]));
|
|
175
|
+
const rawSymbols = dead.deadSymbols
|
|
176
|
+
.map((item) => ({ item, node: nodesById.get(String(item.id)) }))
|
|
177
|
+
.filter((entry) => entry.node)
|
|
178
|
+
.map(({ item, node }) => symbolCandidate(item, node, context));
|
|
179
|
+
const symbolsByFile = new Map();
|
|
180
|
+
for (const candidate of rawSymbols) {
|
|
181
|
+
if (!symbolsByFile.has(candidate.file)) symbolsByFile.set(candidate.file, []);
|
|
182
|
+
symbolsByFile.get(candidate.file).push(candidate);
|
|
183
|
+
}
|
|
184
|
+
const rawFiles = dead.deadFiles.map((item) => fileCandidate(item, symbolsByFile.get(normalizedPath(item.file)) || [], context));
|
|
185
|
+
const raw = [...rawSymbols, ...rawFiles];
|
|
186
|
+
const suppressed = { tests: 0, classified: 0, confidence: 0, path: 0, kind: 0 };
|
|
187
|
+
const candidates = [];
|
|
188
|
+
for (const candidate of raw) {
|
|
189
|
+
const info = classify(candidate.file, sources.get(candidate.file));
|
|
190
|
+
const allowed = pathAllowed(info, { includeTests, includeClassified });
|
|
191
|
+
if (!allowed.ok) { suppressed[allowed.bucket] += 1; continue; }
|
|
192
|
+
if (pathPrefix && candidate.file !== pathPrefix && !candidate.file.startsWith(`${pathPrefix}/`)) { suppressed.path += 1; continue; }
|
|
193
|
+
if (!requestedKinds.has(candidate.kind)) { suppressed.kind += 1; continue; }
|
|
194
|
+
if (CONFIDENCE_RANK[candidate.confidence] > CONFIDENCE_RANK[minConfidence]) { suppressed.confidence += 1; continue; }
|
|
195
|
+
candidates.push(candidate);
|
|
196
|
+
}
|
|
197
|
+
candidates.sort((left, right) =>
|
|
198
|
+
CONFIDENCE_RANK[left.confidence] - CONFIDENCE_RANK[right.confidence]
|
|
199
|
+
|| left.file.localeCompare(right.file)
|
|
200
|
+
|| left.line - right.line
|
|
201
|
+
|| left.id.localeCompare(right.id));
|
|
202
|
+
|
|
203
|
+
const warnings = [{
|
|
204
|
+
code: "STATIC_LIVENESS_IS_NOT_RUNTIME_PROOF",
|
|
205
|
+
message: "No static reference is not proof of dead runtime code. Review every candidate; never bulk-delete or auto-delete.",
|
|
206
|
+
}];
|
|
207
|
+
if (repoSignals.dynamicLoading) warnings.push({
|
|
208
|
+
code: "DYNAMIC_LOADING_PRESENT",
|
|
209
|
+
message: "Dynamic loading exists in the repository; static callers may be incomplete.",
|
|
210
|
+
});
|
|
211
|
+
if (repoSignals.reflection) warnings.push({
|
|
212
|
+
code: "REFLECTION_PRESENT",
|
|
213
|
+
message: "Reflection-like APIs exist in the repository; public or name-addressed symbols may be invoked without graph edges.",
|
|
214
|
+
});
|
|
215
|
+
if (suppressed.confidence) warnings.push({
|
|
216
|
+
code: "LOW_CONFIDENCE_SUPPRESSED",
|
|
217
|
+
message: `${suppressed.confidence} low-confidence candidate(s), including public/framework-sensitive code, were suppressed; set min_confidence=low to review them explicitly.`,
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
return {
|
|
221
|
+
candidates,
|
|
222
|
+
warnings,
|
|
223
|
+
suppressed,
|
|
224
|
+
repoSignals,
|
|
225
|
+
totals: {
|
|
226
|
+
indexedSymbols: dead.stats.symbols,
|
|
227
|
+
indexedFiles: dead.stats.files,
|
|
228
|
+
rawDeadSymbols: rawSymbols.length,
|
|
229
|
+
rawDeadFiles: rawFiles.length,
|
|
230
|
+
reviewCandidates: candidates.length,
|
|
231
|
+
byConfidence: {
|
|
232
|
+
high: candidates.filter((candidate) => candidate.confidence === "high").length,
|
|
233
|
+
medium: candidates.filter((candidate) => candidate.confidence === "medium").length,
|
|
234
|
+
low: candidates.filter((candidate) => candidate.confidence === "low").length,
|
|
235
|
+
},
|
|
236
|
+
},
|
|
237
|
+
policy: {
|
|
238
|
+
verdict: "REVIEW_REQUIRED",
|
|
239
|
+
autoDelete: false,
|
|
240
|
+
minConfidence,
|
|
241
|
+
includeTests,
|
|
242
|
+
includeClassified,
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
}
|
|
@@ -40,6 +40,7 @@ export function computeGoDepFindings({ externalImports = [], goMod = null, nonRu
|
|
|
40
40
|
severity: "low",
|
|
41
41
|
confidence: "medium",
|
|
42
42
|
title: `Unused Go module: ${r.path}`,
|
|
43
|
+
reason: "A direct go.mod requirement has no matching recorded Go import; build-tagged usage remains possible.",
|
|
43
44
|
detail: `"${r.path}" is required (direct) in go.mod but no .go file imports it or any of its packages. Build-tag-guarded files can hide usage — confirm with \`go mod tidy\` before removing.`,
|
|
44
45
|
package: r.path,
|
|
45
46
|
version: r.version,
|
|
@@ -57,6 +58,7 @@ export function computeGoDepFindings({ externalImports = [], goMod = null, nonRu
|
|
|
57
58
|
severity: "medium",
|
|
58
59
|
confidence: "medium",
|
|
59
60
|
title: `Missing Go module: ${pkg}`,
|
|
61
|
+
reason: "A recorded Go import has no matching direct go.mod requirement or replace entry.",
|
|
60
62
|
detail: `"${pkg}" is imported by ${files.length} file(s) but go.mod has no matching require — a replace/workspace/vendor setup, or the module was never added.`,
|
|
61
63
|
package: pkg,
|
|
62
64
|
file: files[0],
|
|
@@ -123,6 +125,7 @@ export function computePyDepFindings({
|
|
|
123
125
|
severity: d.dev ? "info" : "low",
|
|
124
126
|
confidence: "low",
|
|
125
127
|
title: `Unused Python dependency: ${d.name}`,
|
|
128
|
+
reason: "No recorded Python import maps to this declared distribution; import-to-distribution mapping and plugin loading are heuristic.",
|
|
126
129
|
detail: `"${d.name}" is declared but no .py file imports a module that maps to it. Import-name↔package-name mapping is heuristic and plugins/CLI tools load dynamically — review before removing.`,
|
|
127
130
|
package: d.name,
|
|
128
131
|
source: "internal",
|
|
@@ -144,6 +147,9 @@ export function computePyDepFindings({
|
|
|
144
147
|
severity: present ? (testOnly ? "low" : "medium") : "low",
|
|
145
148
|
confidence: present ? "medium" : "low",
|
|
146
149
|
title: `Missing Python dependency: ${u.dist}`,
|
|
150
|
+
reason: present
|
|
151
|
+
? `A recorded Python import maps to "${u.dist}", but no declared distribution covers it.`
|
|
152
|
+
: `A recorded Python import maps to "${u.dist}", but no Python dependency manifest is present; a managed runtime may provide it.`,
|
|
147
153
|
detail: `"${top}" is imported by ${files.length} file(s) but ${present ? "no declared dependency provides it" : "the repo has no Python dependency manifest (requirements.txt / pyproject / Pipfile); a bundled or managed runtime may provide it"}${u.dist !== top ? ` (PyPI package is likely "${u.dist}")` : ""}.${present ? "" : " If this is intentional, declare it under python.managedDependencies in .weavatrix-deps.json."}`,
|
|
148
154
|
package: u.dist,
|
|
149
155
|
file: files[0],
|
|
@@ -36,6 +36,18 @@ const FRAMEWORK_RUNTIME_PEERS = new Map([
|
|
|
36
36
|
["@cloudflare/vite-plugin", ["vite", "wrangler"]],
|
|
37
37
|
]);
|
|
38
38
|
|
|
39
|
+
// Style preprocessors are compiler inputs rather than JavaScript imports. Their presence is proven by
|
|
40
|
+
// source extensions in the same package scope, so do not report them as unused merely because Vite,
|
|
41
|
+
// webpack, etc. load them internally. Keep this list to exact compiler/package contracts.
|
|
42
|
+
const IMPLICIT_STYLE_COMPILERS = new Map([
|
|
43
|
+
["sass", /\.(?:scss|sass)$/i],
|
|
44
|
+
["sass-embedded", /\.(?:scss|sass)$/i],
|
|
45
|
+
["less", /\.less$/i],
|
|
46
|
+
["stylus", /\.styl(?:us)?$/i],
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
const isStylesheetSpecifier = (spec) => /\.(?:css|scss|sass|less|styl(?:us)?)(?:[?#].*)?$/i.test(String(spec || ""));
|
|
50
|
+
|
|
39
51
|
const escRe = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
40
52
|
// word-ish mention: the name not embedded inside a longer identifier/path segment
|
|
41
53
|
const mentioned = (blob, name) => new RegExp(`(^|[^\\w@.-])${escRe(name)}($|[^\\w.-])`).test(blob);
|
|
@@ -47,7 +59,7 @@ const mentioned = (blob, name) => new RegExp(`(^|[^\\w@.-])${escRe(name)}($|[^\\
|
|
|
47
59
|
// configTexts — Map<fileName, text> of root config files + CI workflows (mention scanning)
|
|
48
60
|
export function computeDepFindings({
|
|
49
61
|
externalImports = [], pkg = {}, workspacePkgNames = new Set(), configTexts = new Map(),
|
|
50
|
-
aliases = [], scope = "", manifest = "package.json", nonRuntimeRoots = [],
|
|
62
|
+
aliases = [], scope = "", manifest = "package.json", nonRuntimeRoots = [], sourceFiles = [],
|
|
51
63
|
} = {}) {
|
|
52
64
|
const findings = [];
|
|
53
65
|
const meta = { scope: scope || ".", manifest };
|
|
@@ -101,9 +113,15 @@ export function computeDepFindings({
|
|
|
101
113
|
for (const [provider, peers] of FRAMEWORK_RUNTIME_PEERS) {
|
|
102
114
|
if (allDeclared.has(provider)) for (const peer of peers) frameworkRuntime.add(peer);
|
|
103
115
|
}
|
|
116
|
+
const implicitCompilerUsage = new Set();
|
|
117
|
+
for (const [name, sourcePattern] of IMPLICIT_STYLE_COMPILERS) {
|
|
118
|
+
if (allDeclared.has(name) && sourceFiles.some((file) => sourcePattern.test(String(file || "")))) {
|
|
119
|
+
implicitCompilerUsage.add(name);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
104
122
|
|
|
105
123
|
// ---- usage index from the graph's recorded imports ----
|
|
106
|
-
const usedPackages = new Map(); // pkgName -> { files:Set, lines:Map(file→first line), kinds:Set }
|
|
124
|
+
const usedPackages = new Map(); // pkgName -> { files:Set, lines:Map(file→first line), kinds/specs:Set, typeOnly:boolean }
|
|
107
125
|
let builtinUsed = false;
|
|
108
126
|
for (const e of externalImports) {
|
|
109
127
|
if (e.ecosystem && e.ecosystem !== "npm") continue; // go/python imports have their own checkers
|
|
@@ -112,10 +130,12 @@ export function computeDepFindings({
|
|
|
112
130
|
if (e.builtin) { builtinUsed = true; continue; }
|
|
113
131
|
if (!e.pkg) continue;
|
|
114
132
|
let u = usedPackages.get(e.pkg);
|
|
115
|
-
if (!u) usedPackages.set(e.pkg, (u = { files: new Set(), lines: new Map(), kinds: new Set() }));
|
|
133
|
+
if (!u) usedPackages.set(e.pkg, (u = { files: new Set(), lines: new Map(), kinds: new Set(), specs: new Set(), typeOnly: true }));
|
|
116
134
|
u.files.add(e.file);
|
|
117
135
|
if (!u.lines.has(e.file)) u.lines.set(e.file, e.line || 0);
|
|
118
136
|
u.kinds.add(e.kind);
|
|
137
|
+
u.specs.add(e.spec || e.pkg);
|
|
138
|
+
if (!e.typeOnly) u.typeOnly = false;
|
|
119
139
|
}
|
|
120
140
|
|
|
121
141
|
const scriptBlob = Object.values(pkg.scripts || {}).join("\n");
|
|
@@ -132,7 +152,7 @@ export function computeDepFindings({
|
|
|
132
152
|
for (const [section, deps] of Object.entries(sections)) {
|
|
133
153
|
if (section === "peerDependencies") continue; // peers are consumer-facing contracts, not usage
|
|
134
154
|
for (const name of Object.keys(deps)) {
|
|
135
|
-
if (name === selfName || usedPackages.has(name) || frameworkRuntime.has(name)) continue;
|
|
155
|
+
if (name === selfName || usedPackages.has(name) || frameworkRuntime.has(name) || implicitCompilerUsage.has(name)) continue;
|
|
136
156
|
const tb = typesBase(name);
|
|
137
157
|
if (tb) { // @types/x is used iff x is used (or it types the Node builtins)
|
|
138
158
|
if (tb === "node" ? builtinUsed : usedPackages.has(tb) || frameworkRuntime.has(tb) || mentioned(configBlob, tb)) continue;
|
|
@@ -147,6 +167,7 @@ export function computeDepFindings({
|
|
|
147
167
|
severity: dev ? "info" : "low",
|
|
148
168
|
confidence: dev || ecosystem ? "low" : "medium",
|
|
149
169
|
title: `Unused ${section === "dependencies" ? "dependency" : section.replace(/ies$/, "y")}: ${name}`,
|
|
170
|
+
reason: "No recorded package import, package-script command, recognized config mention, framework peer contract, or implicit style-compiler input uses this declaration.",
|
|
150
171
|
detail: `"${name}" is declared in ${section} but never imported in source, never referenced by a script, and not mentioned in any known config file. Dynamic/config-convention usage can't be fully ruled out — review before removing.`,
|
|
151
172
|
package: name,
|
|
152
173
|
source: "internal",
|
|
@@ -162,12 +183,19 @@ export function computeDepFindings({
|
|
|
162
183
|
const files = [...use.files];
|
|
163
184
|
if (files.every(isNonRuntimeFile) || isGeneratedNapiPackage(name, files)) continue;
|
|
164
185
|
const testOnly = files.every((f) => /(^|[/\\])(test|tests|__tests__|spec|e2e|__mocks__)([/\\]|$)|[._-](test|spec)\.[a-z]+$/i.test(f));
|
|
186
|
+
const stylesheetOnly = [...use.specs].length > 0 && [...use.specs].every(isStylesheetSpecifier);
|
|
187
|
+
const usageReason = stylesheetOnly
|
|
188
|
+
? `Direct stylesheet import(s) resolve through "${name}"; CSS-only imports are build/runtime inputs even without JavaScript bindings.`
|
|
189
|
+
: use.typeOnly
|
|
190
|
+
? `Direct type-only import(s) resolve through "${name}"; the compiler still requires a declared package.`
|
|
191
|
+
: `Direct source import(s) resolve through "${name}", but the nearest package manifest does not declare it.`;
|
|
165
192
|
findings.push(makeFinding({
|
|
166
193
|
category: "unused",
|
|
167
194
|
rule: "missing-dep",
|
|
168
195
|
severity: testOnly ? "low" : "medium",
|
|
169
196
|
confidence: "high",
|
|
170
197
|
title: `Missing dependency: ${name}`,
|
|
198
|
+
reason: usageReason,
|
|
171
199
|
detail: `"${name}" is imported by ${files.length} file(s) in scope ${meta.scope} but not declared in ${manifest} — it only works via a transitive install (phantom dependency) and can break on any lockfile change.`,
|
|
172
200
|
package: name,
|
|
173
201
|
file: files[0],
|
|
@@ -191,6 +219,7 @@ export function computeDepFindings({
|
|
|
191
219
|
severity: "info",
|
|
192
220
|
confidence: "high",
|
|
193
221
|
title: `Duplicate declaration: ${name}`,
|
|
222
|
+
reason: `The same package is declared in multiple manifest sections: ${ss.join(" + ")}.`,
|
|
194
223
|
detail: `"${name}" is declared in ${ss.join(" + ")} — npm resolves one of them; keep a single section.`,
|
|
195
224
|
package: name,
|
|
196
225
|
source: "internal",
|
|
@@ -244,16 +273,21 @@ const ownsFile = (scope, file) => !scope || file === scope || String(file || "")
|
|
|
244
273
|
// Node's package scope and prevents nested Next/Vite apps from inheriting the root manifest by accident.
|
|
245
274
|
export function computeScopedDepFindings({
|
|
246
275
|
externalImports = [], packageScopes = [], workspacePkgNames = new Set(), configTexts = new Map(),
|
|
247
|
-
nonRuntimeRoots = [],
|
|
276
|
+
nonRuntimeRoots = [], sourceFiles = [],
|
|
248
277
|
} = {}) {
|
|
249
278
|
const scopes = packageScopes.length
|
|
250
279
|
? packageScopes.map((s) => ({ ...s, root: normScope(s.root) })).sort((a, b) => b.root.length - a.root.length)
|
|
251
280
|
: [{ root: "", manifest: "package.json", pkg: {}, aliases: [] }];
|
|
252
281
|
const importsByScope = new Map(scopes.map((s) => [s, []]));
|
|
282
|
+
const sourceFilesByScope = new Map(scopes.map((s) => [s, []]));
|
|
253
283
|
for (const e of externalImports) {
|
|
254
284
|
const owner = scopes.find((s) => ownsFile(s.root, e.file)) || scopes[scopes.length - 1];
|
|
255
285
|
importsByScope.get(owner).push(e);
|
|
256
286
|
}
|
|
287
|
+
for (const file of sourceFiles) {
|
|
288
|
+
const owner = scopes.find((s) => ownsFile(s.root, file)) || scopes[scopes.length - 1];
|
|
289
|
+
sourceFilesByScope.get(owner).push(file);
|
|
290
|
+
}
|
|
257
291
|
const configOwner = new Map();
|
|
258
292
|
for (const [file] of configTexts) configOwner.set(file, scopes.find((scope) => ownsFile(scope.root, file)) || scopes[scopes.length - 1]);
|
|
259
293
|
const findings = [], usedPackages = new Map(), declared = new Set();
|
|
@@ -262,7 +296,7 @@ export function computeScopedDepFindings({
|
|
|
262
296
|
const r = computeDepFindings({
|
|
263
297
|
externalImports: importsByScope.get(s), pkg: s.pkg || {}, workspacePkgNames, configTexts: scopeConfig,
|
|
264
298
|
aliases: s.aliases || [], scope: s.root, manifest: s.manifest || (s.root ? `${s.root}/package.json` : "package.json"),
|
|
265
|
-
nonRuntimeRoots,
|
|
299
|
+
nonRuntimeRoots, sourceFiles: sourceFilesByScope.get(s),
|
|
266
300
|
});
|
|
267
301
|
findings.push(...r.findings);
|
|
268
302
|
for (const [name, use] of r.usedPackages) usedPackages.set(`${s.root || "."}:${name}`, use);
|
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
// built from EXTRACTED import edges → high confidence.
|
|
8
8
|
import { makeFinding } from "./findings.js";
|
|
9
9
|
import { ENTRY_FILE } from "./dead-check.js";
|
|
10
|
+
import { formatRepresentativeCycle } from "./cycle-route.js";
|
|
10
11
|
const TEST_FILE_RE = /(^|[/])(test|tests|__tests__|spec|e2e|__mocks__)([/]|$)|[._-](test|spec)\.[a-z0-9]+$/i;
|
|
11
12
|
// config/data/docs: never "orphans" — nothing imports them by design
|
|
12
13
|
const NON_CODE_RE = /\.(json|ya?ml|sh|ps1|md|txt|html?|css|scss|less)$|(^|[/])(dockerfile|containerfile)/i;
|
|
13
|
-
|
|
14
14
|
const ep = (v) => String(v && typeof v === "object" ? v.id : v);
|
|
15
15
|
const fileOf = (v) => { const s = ep(v); const h = s.indexOf("#"); return h < 0 ? s : s.slice(0, h); };
|
|
16
16
|
const dirOf = (f) => (f.includes("/") ? f.slice(0, f.lastIndexOf("/")) : "");
|
|
@@ -63,7 +63,6 @@ export function buildFileImportGraph(graph, { includeTypeOnly = false, includeCo
|
|
|
63
63
|
compileTimeEdges: pureCompileTimeEdges,
|
|
64
64
|
};
|
|
65
65
|
}
|
|
66
|
-
|
|
67
66
|
// Iterative Tarjan (deep graphs must not blow the JS stack). Returns only non-trivial SCCs (size > 1).
|
|
68
67
|
export function findSccs(adj) {
|
|
69
68
|
const index = new Map(), low = new Map(), onStack = new Set(), S = [];
|
|
@@ -99,7 +98,6 @@ export function findSccs(adj) {
|
|
|
99
98
|
}
|
|
100
99
|
return sccs;
|
|
101
100
|
}
|
|
102
|
-
|
|
103
101
|
// One representative (shortest) cycle through an SCC's lexicographically-first file — a readable path,
|
|
104
102
|
// not the full Johnson enumeration (which explodes combinatorially on big tangles).
|
|
105
103
|
export function representativeCycle(adj, scc) {
|
|
@@ -109,7 +107,7 @@ export function representativeCycle(adj, scc) {
|
|
|
109
107
|
const q = [start];
|
|
110
108
|
while (q.length) {
|
|
111
109
|
const cur = q.shift();
|
|
112
|
-
for (const nxt of adj.get(cur) || []) {
|
|
110
|
+
for (const nxt of [...(adj.get(cur) || [])].sort()) {
|
|
113
111
|
if (nxt === start) {
|
|
114
112
|
const path = [];
|
|
115
113
|
for (let c = cur; c != null; c = prev.get(c)) path.push(c);
|
|
@@ -120,9 +118,9 @@ export function representativeCycle(adj, scc) {
|
|
|
120
118
|
q.push(nxt);
|
|
121
119
|
}
|
|
122
120
|
}
|
|
123
|
-
|
|
121
|
+
const fallback = [...scc].sort();
|
|
122
|
+
return [...fallback, fallback[0]]; // unreachable in a true SCC; deterministic safe fallback
|
|
124
123
|
}
|
|
125
|
-
|
|
126
124
|
// Orphans: file nodes with ZERO non-contains graph degree (nothing in, nothing out — imports, calls,
|
|
127
125
|
// references all collapsed to files). Entries/tests/config-data are exempt. A file that DOES import
|
|
128
126
|
// npm packages (externalImports) is a working script, not an island — confidence drops, not the verdict.
|
|
@@ -146,7 +144,6 @@ export function findOrphans(graph, { entrySet = new Set(), externalImportFiles =
|
|
|
146
144
|
}
|
|
147
145
|
return out;
|
|
148
146
|
}
|
|
149
|
-
|
|
150
147
|
// Boundary DSL — the useful subset of depcruise's forbidden rules, as plain JSON globs:
|
|
151
148
|
// { "forbidden": [{ "name", "comment"?, "severity"?, "from": "main/**", "to": "renderer/**" }],
|
|
152
149
|
// "allowedOnly":[{ "name", "comment"?, "severity"?, "from": "src/ui/**", "to": ["src/ui/**", "src/shared/**"] }] }
|
|
@@ -180,13 +177,16 @@ export function computeStructureFindings(graph, { rules = {}, entrySet = new Set
|
|
|
180
177
|
const sccs = findSccs(adj).sort((a, b) => b.length - a.length);
|
|
181
178
|
for (const scc of sccs.slice(0, MAX_CYCLE_FINDINGS)) {
|
|
182
179
|
const cycle = representativeCycle(adj, scc);
|
|
180
|
+
const cycleRoute = formatRepresentativeCycle(cycle);
|
|
183
181
|
findings.push(makeFinding({
|
|
184
182
|
category: "structure",
|
|
185
183
|
rule: "circular-dep",
|
|
186
184
|
severity: scc.length > 4 ? "high" : "medium",
|
|
187
185
|
confidence: "high",
|
|
188
186
|
title: `Circular dependency: ${scc.length} files`,
|
|
189
|
-
detail: `${
|
|
187
|
+
detail: `${cycleRoute}${scc.length + 1 > cycle.length ? ` (representative loop; the tangle spans ${scc.length} files)` : ""}. Break the cycle by extracting the shared piece or inverting one import.`,
|
|
188
|
+
cycleRoute,
|
|
189
|
+
cycleMembers: [...scc].sort(),
|
|
190
190
|
file: cycle[0],
|
|
191
191
|
graphNodeId: cycle[0],
|
|
192
192
|
evidence: cycle.map((f) => ({ file: f, line: 0, snippet: "" })),
|
|
@@ -207,6 +207,7 @@ export function computeStructureFindings(graph, { rules = {}, entrySet = new Set
|
|
|
207
207
|
};
|
|
208
208
|
for (const scc of compileTimeCouplings.slice(0, MAX_CYCLE_FINDINGS)) {
|
|
209
209
|
const cycle = representativeCycle(allAdj, scc);
|
|
210
|
+
const cycleRoute = formatRepresentativeCycle(cycle);
|
|
210
211
|
const runtimeInside = edgeCountIn(scc, edges);
|
|
211
212
|
const typeInside = edgeCountIn(scc, typeOnlyEdges);
|
|
212
213
|
const compileInside = edgeCountIn(scc, compileOnlyEdges);
|
|
@@ -220,7 +221,9 @@ export function computeStructureFindings(graph, { rules = {}, entrySet = new Set
|
|
|
220
221
|
title: `${containsRuntimeCycle
|
|
221
222
|
? (typeSpecific ? "Type imports expand dependency coupling" : "Compile-time edges expand dependency coupling")
|
|
222
223
|
: (typeSpecific ? "Type-induced dependency cycle (no runtime cycle)" : "Compile-time dependency cycle (no runtime cycle)")}: ${scc.length} files`,
|
|
223
|
-
detail: `${
|
|
224
|
+
detail: `${cycleRoute}. This strongly-connected group needs compile-time-only edges to close; it contains ${runtimeInside} runtime edge(s), ${typeInside} type-only edge(s), and ${compileInside} compile-only edge(s)${containsRuntimeCycle ? ", with a smaller runtime cycle reported separately" : ", while its runtime import graph is acyclic"}. Treat this as design coupling, not an initialization-order failure.`,
|
|
225
|
+
cycleRoute,
|
|
226
|
+
cycleMembers: [...scc].sort(),
|
|
224
227
|
file: cycle[0],
|
|
225
228
|
graphNodeId: cycle[0],
|
|
226
229
|
evidence: cycle.map((f) => ({ file: f, line: 0, snippet: "" })),
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// duplicates.compute.js — fragment extraction, inverted-index pairing, and the computeDuplicates
|
|
2
2
|
// pipeline (split from duplicates.js; see the facade there for the full algorithm overview).
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
|
-
import { isTestPath } from "../graph/graph-filter.js";
|
|
5
4
|
import { createRepoBoundary } from "../repo-path.js";
|
|
5
|
+
import { createPathClassifier, hasPathClass } from "../path-classification.js";
|
|
6
6
|
import { listRepoFiles } from "./internal-audit.collect.js";
|
|
7
7
|
import { stripNonCode, bodyEndLineCount, tokenize, fingerprints, extractLargeStrings } from "./duplicates.tokenize.js";
|
|
8
8
|
|
|
@@ -113,9 +113,19 @@ function symbolRanges(graph) {
|
|
|
113
113
|
return byFile;
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
+
function loadGraph(graphInput) {
|
|
117
|
+
if (graphInput && typeof graphInput === "object" && !Array.isArray(graphInput)) {
|
|
118
|
+
return { nodes: Array.isArray(graphInput.nodes) ? graphInput.nodes : [] };
|
|
119
|
+
}
|
|
120
|
+
if (typeof graphInput !== "string" || !graphInput) throw new TypeError("graph path or graph object is required");
|
|
121
|
+
const parsed = JSON.parse(readFileSync(graphInput, "utf8"));
|
|
122
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new TypeError("graph must be an object");
|
|
123
|
+
return { nodes: Array.isArray(parsed.nodes) ? parsed.nodes : [] };
|
|
124
|
+
}
|
|
125
|
+
|
|
116
126
|
export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
117
127
|
const includeStrings = !!opts.includeStrings;
|
|
118
|
-
const graph =
|
|
128
|
+
const graph = loadGraph(graphJsonPath);
|
|
119
129
|
const byFile = symbolRanges(graph);
|
|
120
130
|
// total symbol nodes in the graph — 0 means a file-only graph (built by an older builder before
|
|
121
131
|
// symbol extraction, or a stale graph): clone detection has nothing to work with, and the UI must
|
|
@@ -126,6 +136,18 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
126
136
|
for (const [file, arr] of byFile) if (allowedFiles.has(file)) graphSymbols += arr.length;
|
|
127
137
|
const frags = [];
|
|
128
138
|
const boundary = createRepoBoundary(repoPath);
|
|
139
|
+
const classifier = createPathClassifier(repoPath);
|
|
140
|
+
const classificationByFile = new Map();
|
|
141
|
+
const classify = (file, content) => {
|
|
142
|
+
if (!classificationByFile.has(file)) classificationByFile.set(file, classifier.explain(file, { content }));
|
|
143
|
+
return classificationByFile.get(file);
|
|
144
|
+
};
|
|
145
|
+
const classificationFields = (info) => ({
|
|
146
|
+
test: hasPathClass(info, "test", "e2e"),
|
|
147
|
+
classes: info.classes,
|
|
148
|
+
excluded: info.excluded,
|
|
149
|
+
matchedRule: info.matchedRule,
|
|
150
|
+
});
|
|
129
151
|
for (const [file, syms] of byFile) {
|
|
130
152
|
if (!allowedFiles.has(file)) continue;
|
|
131
153
|
const resolved = boundary.resolve(file);
|
|
@@ -134,6 +156,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
134
156
|
let lines;
|
|
135
157
|
try { lines = readFileSync(full, "utf8").split(/\r?\n/); } catch { continue; }
|
|
136
158
|
const py = /\.py$/i.test(file);
|
|
159
|
+
const pathInfo = classify(file, lines.join("\n"));
|
|
137
160
|
for (let i = 0; i < syms.length; i++) {
|
|
138
161
|
const start = syms[i].line;
|
|
139
162
|
// hard cap first (next symbol, or EOF, whichever comes first, ≤ MAX_BODY_LINES) …
|
|
@@ -150,7 +173,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
150
173
|
frags.push({
|
|
151
174
|
id: syms[i].id, label: syms[i].label, file, start, end,
|
|
152
175
|
n: toks.strict.length,
|
|
153
|
-
|
|
176
|
+
...classificationFields(pathInfo),
|
|
154
177
|
fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
|
|
155
178
|
});
|
|
156
179
|
}
|
|
@@ -164,7 +187,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
164
187
|
if (toks.strict.length < FLOOR_TOKENS) return;
|
|
165
188
|
frags.push({
|
|
166
189
|
id: `${file}#str@${startLine}`, label: `${base}:${startLine} ~${endLine - startLine + 1}-line string`,
|
|
167
|
-
file, start: startLine, end: endLine, n: toks.strict.length,
|
|
190
|
+
file, start: startLine, end: endLine, n: toks.strict.length, ...classificationFields(pathInfo), kind: "string",
|
|
168
191
|
fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
|
|
169
192
|
});
|
|
170
193
|
};
|
|
@@ -188,13 +211,15 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
188
211
|
// ---- windowed fragments for symbol-less file types (CSS/HTML/MD/…), found by walking the repo (NOT the
|
|
189
212
|
// graph — assets are dedup-only). Each file is sliced into WINDOW_LINES blocks and fingerprinted.
|
|
190
213
|
const symFiles = new Set(byFile.keys());
|
|
191
|
-
const
|
|
214
|
+
const allAssetFiles = repoFiles.filter((file) => WINDOW_EXTS.test(file));
|
|
215
|
+
const assetFiles = allAssetFiles.slice(0, MAX_ASSET_FILES);
|
|
192
216
|
for (const file of assetFiles) {
|
|
193
217
|
if (symFiles.has(file)) continue;
|
|
194
218
|
const resolved = boundary.resolve(file);
|
|
195
219
|
if (!resolved.ok) continue;
|
|
196
220
|
let lines;
|
|
197
221
|
try { lines = readFileSync(resolved.path, "utf8").split(/\r?\n/); } catch { continue; }
|
|
222
|
+
const pathInfo = classify(file, lines.join("\n"));
|
|
198
223
|
for (let start = 1; start <= lines.length; start += WINDOW_LINES) {
|
|
199
224
|
const end = Math.min(start + WINDOW_LINES - 1, lines.length);
|
|
200
225
|
if (end - start < 2) continue;
|
|
@@ -202,7 +227,7 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
202
227
|
if (toks.strict.length < FLOOR_TOKENS) continue;
|
|
203
228
|
frags.push({
|
|
204
229
|
id: `${file}#win@${start}`, label: `${file.split("/").pop()}:${start}-${end}`, file, start, end,
|
|
205
|
-
n: toks.strict.length,
|
|
230
|
+
n: toks.strict.length, ...classificationFields(pathInfo),
|
|
206
231
|
fp: { strict: fingerprints(toks.strict), renamed: fingerprints(toks.renamed) },
|
|
207
232
|
});
|
|
208
233
|
}
|
|
@@ -211,5 +236,20 @@ export function computeDuplicates(repoPath, graphJsonPath, opts = {}) {
|
|
|
211
236
|
const nameTwins = opts.nameTwins ? computeNameTwins(frags) : null;
|
|
212
237
|
// fp sets are worker-internal — strip them from the payload that crosses the thread boundary
|
|
213
238
|
const slim = frags.map(({ fp, ...rest }) => rest);
|
|
214
|
-
return {
|
|
239
|
+
return {
|
|
240
|
+
ok: true,
|
|
241
|
+
frags: slim,
|
|
242
|
+
modes,
|
|
243
|
+
...(nameTwins ? { nameTwins } : {}),
|
|
244
|
+
graphSymbols,
|
|
245
|
+
completeness: {
|
|
246
|
+
assetFiles: {
|
|
247
|
+
total: allAssetFiles.length,
|
|
248
|
+
scanned: assetFiles.length,
|
|
249
|
+
truncated: allAssetFiles.length > assetFiles.length,
|
|
250
|
+
},
|
|
251
|
+
nameTwinsTruncated: !!nameTwins && nameTwins.length >= 200,
|
|
252
|
+
},
|
|
253
|
+
floors: { tokens: FLOOR_TOKENS, sim: FLOOR_SIM * 100 },
|
|
254
|
+
};
|
|
215
255
|
}
|