seer-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.vscode/settings.json +3 -0
- package/LICENSE +176 -0
- package/README.md +272 -0
- package/README_dev.md +199 -0
- package/dist/bundle/ci.d.ts +47 -0
- package/dist/bundle/ci.d.ts.map +1 -0
- package/dist/bundle/ci.js +113 -0
- package/dist/bundle/ci.js.map +1 -0
- package/dist/bundle/contract.d.ts +111 -0
- package/dist/bundle/contract.d.ts.map +1 -0
- package/dist/bundle/contract.js +352 -0
- package/dist/bundle/contract.js.map +1 -0
- package/dist/bundle/export.d.ts +36 -0
- package/dist/bundle/export.d.ts.map +1 -0
- package/dist/bundle/export.js +152 -0
- package/dist/bundle/export.js.map +1 -0
- package/dist/bundle/external.d.ts +66 -0
- package/dist/bundle/external.d.ts.map +1 -0
- package/dist/bundle/external.js +238 -0
- package/dist/bundle/external.js.map +1 -0
- package/dist/bundle/format.d.ts +94 -0
- package/dist/bundle/format.d.ts.map +1 -0
- package/dist/bundle/format.js +42 -0
- package/dist/bundle/format.js.map +1 -0
- package/dist/bundle/import.d.ts +49 -0
- package/dist/bundle/import.d.ts.map +1 -0
- package/dist/bundle/import.js +116 -0
- package/dist/bundle/import.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +1402 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/init.d.ts +48 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +284 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/db/schema.d.ts +3 -0
- package/dist/db/schema.d.ts.map +1 -0
- package/dist/db/schema.js +616 -0
- package/dist/db/schema.js.map +1 -0
- package/dist/db/store.d.ts +1011 -0
- package/dist/db/store.d.ts.map +1 -0
- package/dist/db/store.js +3888 -0
- package/dist/db/store.js.map +1 -0
- package/dist/graph/pagerank.d.ts +9 -0
- package/dist/graph/pagerank.d.ts.map +1 -0
- package/dist/graph/pagerank.js +47 -0
- package/dist/graph/pagerank.js.map +1 -0
- package/dist/indexer/architecture.d.ts +72 -0
- package/dist/indexer/architecture.d.ts.map +1 -0
- package/dist/indexer/architecture.js +112 -0
- package/dist/indexer/architecture.js.map +1 -0
- package/dist/indexer/behavior.d.ts +75 -0
- package/dist/indexer/behavior.d.ts.map +1 -0
- package/dist/indexer/behavior.js +395 -0
- package/dist/indexer/behavior.js.map +1 -0
- package/dist/indexer/boundaries.d.ts +60 -0
- package/dist/indexer/boundaries.d.ts.map +1 -0
- package/dist/indexer/boundaries.js +366 -0
- package/dist/indexer/boundaries.js.map +1 -0
- package/dist/indexer/churn.d.ts +15 -0
- package/dist/indexer/churn.d.ts.map +1 -0
- package/dist/indexer/churn.js +49 -0
- package/dist/indexer/churn.js.map +1 -0
- package/dist/indexer/classify.d.ts +9 -0
- package/dist/indexer/classify.d.ts.map +1 -0
- package/dist/indexer/classify.js +90 -0
- package/dist/indexer/classify.js.map +1 -0
- package/dist/indexer/context.d.ts +176 -0
- package/dist/indexer/context.d.ts.map +1 -0
- package/dist/indexer/context.js +193 -0
- package/dist/indexer/context.js.map +1 -0
- package/dist/indexer/continuity.d.ts +67 -0
- package/dist/indexer/continuity.d.ts.map +1 -0
- package/dist/indexer/continuity.js +288 -0
- package/dist/indexer/continuity.js.map +1 -0
- package/dist/indexer/detectchanges.d.ts +32 -0
- package/dist/indexer/detectchanges.d.ts.map +1 -0
- package/dist/indexer/detectchanges.js +74 -0
- package/dist/indexer/detectchanges.js.map +1 -0
- package/dist/indexer/discovery.d.ts +37 -0
- package/dist/indexer/discovery.d.ts.map +1 -0
- package/dist/indexer/discovery.js +136 -0
- package/dist/indexer/discovery.js.map +1 -0
- package/dist/indexer/externaldeps.d.ts +18 -0
- package/dist/indexer/externaldeps.d.ts.map +1 -0
- package/dist/indexer/externaldeps.js +288 -0
- package/dist/indexer/externaldeps.js.map +1 -0
- package/dist/indexer/freshness.d.ts +48 -0
- package/dist/indexer/freshness.d.ts.map +1 -0
- package/dist/indexer/freshness.js +128 -0
- package/dist/indexer/freshness.js.map +1 -0
- package/dist/indexer/git.d.ts +144 -0
- package/dist/indexer/git.d.ts.map +1 -0
- package/dist/indexer/git.js +444 -0
- package/dist/indexer/git.js.map +1 -0
- package/dist/indexer/index.d.ts +145 -0
- package/dist/indexer/index.d.ts.map +1 -0
- package/dist/indexer/index.js +930 -0
- package/dist/indexer/index.js.map +1 -0
- package/dist/indexer/modules.d.ts +62 -0
- package/dist/indexer/modules.d.ts.map +1 -0
- package/dist/indexer/modules.js +293 -0
- package/dist/indexer/modules.js.map +1 -0
- package/dist/indexer/preflight.d.ts +154 -0
- package/dist/indexer/preflight.d.ts.map +1 -0
- package/dist/indexer/preflight.js +399 -0
- package/dist/indexer/preflight.js.map +1 -0
- package/dist/indexer/protoScanner.d.ts +34 -0
- package/dist/indexer/protoScanner.d.ts.map +1 -0
- package/dist/indexer/protoScanner.js +133 -0
- package/dist/indexer/protoScanner.js.map +1 -0
- package/dist/indexer/risk.d.ts +115 -0
- package/dist/indexer/risk.d.ts.map +1 -0
- package/dist/indexer/risk.js +194 -0
- package/dist/indexer/risk.js.map +1 -0
- package/dist/indexer/serviceHostScanner.d.ts +25 -0
- package/dist/indexer/serviceHostScanner.d.ts.map +1 -0
- package/dist/indexer/serviceHostScanner.js +95 -0
- package/dist/indexer/serviceHostScanner.js.map +1 -0
- package/dist/indexer/serviceLinks.d.ts +105 -0
- package/dist/indexer/serviceLinks.d.ts.map +1 -0
- package/dist/indexer/serviceLinks.js +509 -0
- package/dist/indexer/serviceLinks.js.map +1 -0
- package/dist/indexer/shapehash.d.ts +98 -0
- package/dist/indexer/shapehash.d.ts.map +1 -0
- package/dist/indexer/shapehash.js +354 -0
- package/dist/indexer/shapehash.js.map +1 -0
- package/dist/indexer/skeleton.d.ts +15 -0
- package/dist/indexer/skeleton.d.ts.map +1 -0
- package/dist/indexer/skeleton.js +136 -0
- package/dist/indexer/skeleton.js.map +1 -0
- package/dist/indexer/symbolhistory.d.ts +41 -0
- package/dist/indexer/symbolhistory.d.ts.map +1 -0
- package/dist/indexer/symbolhistory.js +124 -0
- package/dist/indexer/symbolhistory.js.map +1 -0
- package/dist/indexer/watcher.d.ts +68 -0
- package/dist/indexer/watcher.d.ts.map +1 -0
- package/dist/indexer/watcher.js +179 -0
- package/dist/indexer/watcher.js.map +1 -0
- package/dist/mcp/server.d.ts +80 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +1610 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/parser/index.d.ts +8 -0
- package/dist/parser/index.d.ts.map +1 -0
- package/dist/parser/index.js +33 -0
- package/dist/parser/index.js.map +1 -0
- package/dist/parser/languages/cpp.d.ts +3 -0
- package/dist/parser/languages/cpp.d.ts.map +1 -0
- package/dist/parser/languages/cpp.js +350 -0
- package/dist/parser/languages/cpp.js.map +1 -0
- package/dist/parser/languages/csharp.d.ts +3 -0
- package/dist/parser/languages/csharp.d.ts.map +1 -0
- package/dist/parser/languages/csharp.js +239 -0
- package/dist/parser/languages/csharp.js.map +1 -0
- package/dist/parser/languages/go.d.ts +3 -0
- package/dist/parser/languages/go.d.ts.map +1 -0
- package/dist/parser/languages/go.js +259 -0
- package/dist/parser/languages/go.js.map +1 -0
- package/dist/parser/languages/java.d.ts +3 -0
- package/dist/parser/languages/java.d.ts.map +1 -0
- package/dist/parser/languages/java.js +391 -0
- package/dist/parser/languages/java.js.map +1 -0
- package/dist/parser/languages/python.d.ts +3 -0
- package/dist/parser/languages/python.d.ts.map +1 -0
- package/dist/parser/languages/python.js +396 -0
- package/dist/parser/languages/python.js.map +1 -0
- package/dist/parser/languages/rust.d.ts +3 -0
- package/dist/parser/languages/rust.d.ts.map +1 -0
- package/dist/parser/languages/rust.js +159 -0
- package/dist/parser/languages/rust.js.map +1 -0
- package/dist/parser/languages/typescript.d.ts +3 -0
- package/dist/parser/languages/typescript.d.ts.map +1 -0
- package/dist/parser/languages/typescript.js +1442 -0
- package/dist/parser/languages/typescript.js.map +1 -0
- package/dist/parser/parserContext.d.ts +77 -0
- package/dist/parser/parserContext.d.ts.map +1 -0
- package/dist/parser/parserContext.js +354 -0
- package/dist/parser/parserContext.js.map +1 -0
- package/dist/parser/walker.d.ts +81 -0
- package/dist/parser/walker.d.ts.map +1 -0
- package/dist/parser/walker.js +217 -0
- package/dist/parser/walker.js.map +1 -0
- package/dist/parser/worker.d.ts +66 -0
- package/dist/parser/worker.d.ts.map +1 -0
- package/dist/parser/worker.js +129 -0
- package/dist/parser/worker.js.map +1 -0
- package/dist/parser/workerpool.d.ts +107 -0
- package/dist/parser/workerpool.d.ts.map +1 -0
- package/dist/parser/workerpool.js +383 -0
- package/dist/parser/workerpool.js.map +1 -0
- package/dist/scip/format.d.ts +87 -0
- package/dist/scip/format.d.ts.map +1 -0
- package/dist/scip/format.js +31 -0
- package/dist/scip/format.js.map +1 -0
- package/dist/scip/import.d.ts +37 -0
- package/dist/scip/import.d.ts.map +1 -0
- package/dist/scip/import.js +180 -0
- package/dist/scip/import.js.map +1 -0
- package/dist/types.d.ts +392 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +4 -0
- package/dist/types.js.map +1 -0
- package/docs/architecture.md +105 -0
- package/docs/benchmarks/methodology.md +134 -0
- package/docs/benchmarks/raw-results.md +71 -0
- package/docs/benchmarks.md +74 -0
- package/docs/cli.md +148 -0
- package/docs/examples/behavior-tests.md +70 -0
- package/docs/examples/change-history.md +85 -0
- package/docs/examples/pre-edit-context.md +81 -0
- package/docs/examples/service-links.md +88 -0
- package/docs/examples.md +80 -0
- package/docs/faq.md +70 -0
- package/docs/internals.md +104 -0
- package/docs/languages.md +70 -0
- package/docs/limits.md +52 -0
- package/docs/mcp.md +199 -0
- package/docs/quickstart.md +119 -0
- package/docs/testing.md +123 -0
- package/docs/tools.md +115 -0
- package/package.json +52 -0
- package/research-codebase.md +578 -0
- package/seer-cli-docs.md +326 -0
- package/seer-master-guide.md +246 -0
- package/src/bundle/ci.ts +141 -0
- package/src/bundle/contract.ts +387 -0
- package/src/bundle/export.ts +175 -0
- package/src/bundle/external.ts +285 -0
- package/src/bundle/format.ts +92 -0
- package/src/bundle/import.ts +157 -0
- package/src/cli/index.ts +1249 -0
- package/src/cli/init.ts +389 -0
- package/src/db/schema.ts +614 -0
- package/src/db/store.ts +4306 -0
- package/src/graph/pagerank.ts +53 -0
- package/src/indexer/architecture.ts +148 -0
- package/src/indexer/behavior.ts +466 -0
- package/src/indexer/boundaries.ts +374 -0
- package/src/indexer/churn.ts +58 -0
- package/src/indexer/classify.ts +96 -0
- package/src/indexer/context.ts +340 -0
- package/src/indexer/continuity.ts +322 -0
- package/src/indexer/detectchanges.ts +94 -0
- package/src/indexer/discovery.ts +176 -0
- package/src/indexer/externaldeps.ts +243 -0
- package/src/indexer/freshness.ts +166 -0
- package/src/indexer/git.ts +453 -0
- package/src/indexer/index.ts +1092 -0
- package/src/indexer/modules.ts +358 -0
- package/src/indexer/preflight.ts +548 -0
- package/src/indexer/protoScanner.ts +147 -0
- package/src/indexer/risk.ts +304 -0
- package/src/indexer/serviceHostScanner.ts +92 -0
- package/src/indexer/serviceLinks.ts +543 -0
- package/src/indexer/shapehash.ts +370 -0
- package/src/indexer/skeleton.ts +169 -0
- package/src/indexer/symbolhistory.ts +172 -0
- package/src/indexer/watcher.ts +206 -0
- package/src/mcp/server.ts +1659 -0
- package/src/parser/index.ts +37 -0
- package/src/parser/languages/cpp.ts +361 -0
- package/src/parser/languages/csharp.ts +235 -0
- package/src/parser/languages/go.ts +259 -0
- package/src/parser/languages/java.ts +382 -0
- package/src/parser/languages/python.ts +370 -0
- package/src/parser/languages/rust.ts +164 -0
- package/src/parser/languages/typescript.ts +1435 -0
- package/src/parser/parserContext.ts +392 -0
- package/src/parser/walker.ts +306 -0
- package/src/parser/worker.ts +181 -0
- package/src/parser/workerpool.ts +448 -0
- package/src/scip/format.ts +83 -0
- package/src/scip/import.ts +216 -0
- package/src/types.ts +457 -0
- package/tests/benchmark-service-links.ts +244 -0
- package/tests/bug-regressions.ts +626 -0
- package/tests/filters.ts +264 -0
- package/tests/fixtures/Counter.tsx +38 -0
- package/tests/fixtures/caller.ts +7 -0
- package/tests/fixtures/collisions.ts +23 -0
- package/tests/fixtures/local_helper.ts +5 -0
- package/tests/fixtures/overloads.java +17 -0
- package/tests/fixtures/remote_helper.ts +4 -0
- package/tests/fixtures/sample.c +15 -0
- package/tests/fixtures/sample.cpp +47 -0
- package/tests/fixtures/sample.cs +62 -0
- package/tests/fixtures/sample.go +68 -0
- package/tests/fixtures/sample.h +30 -0
- package/tests/fixtures/sample.java +85 -0
- package/tests/fixtures/sample.py +46 -0
- package/tests/fixtures/sample.rs +78 -0
- package/tests/fixtures/sample.ts +76 -0
- package/tests/fixtures-service/HttpClients.cs +30 -0
- package/tests/fixtures-service/HttpClients.java +24 -0
- package/tests/fixtures-service/billing.ts +15 -0
- package/tests/fixtures-service/docker-compose.yml +15 -0
- package/tests/fixtures-service/gateway.ts +10 -0
- package/tests/fixtures-service/get_user.ts +11 -0
- package/tests/fixtures-service/graphql_client.ts +63 -0
- package/tests/fixtures-service/graphql_server.ts +30 -0
- package/tests/fixtures-service/grpc_client.go +30 -0
- package/tests/fixtures-service/http_clients.go +23 -0
- package/tests/fixtures-service/http_clients.py +38 -0
- package/tests/fixtures-service/http_clients.ts +49 -0
- package/tests/fixtures-service/k8s/payment-service.yaml +22 -0
- package/tests/fixtures-service/k8s_calls.ts +20 -0
- package/tests/fixtures-service/messaging.ts +87 -0
- package/tests/fixtures-service/trpc_client.ts +39 -0
- package/tests/fixtures-service/trpc_server.ts +39 -0
- package/tests/fixtures-service/user_service.proto +33 -0
- package/tests/fixtures-trackcd/Cargo.toml +11 -0
- package/tests/fixtures-trackcd/SpringController.java +36 -0
- package/tests/fixtures-trackcd/auth_service.ts +19 -0
- package/tests/fixtures-trackcd/complex_module.py +50 -0
- package/tests/fixtures-trackcd/express_app.js +30 -0
- package/tests/fixtures-trackcd/fastapi_app.py +49 -0
- package/tests/fixtures-trackcd/fastify_object_routes.js +32 -0
- package/tests/fixtures-trackcd/go.mod +8 -0
- package/tests/fixtures-trackcd/package.json +15 -0
- package/tests/fixtures-trackcd/requirements.txt +4 -0
- package/tests/fixtures-trackcd/tests/auth_service.test.ts +13 -0
- package/tests/fixtures-tracke/auth/AuthService.ts +23 -0
- package/tests/fixtures-tracke/auth/crypto.ts +7 -0
- package/tests/fixtures-tracke/billing/Billing.ts +20 -0
- package/tests/fixtures-tracke/billing/Invoice.ts +10 -0
- package/tests/fixtures-tracke/billing/server.ts +17 -0
- package/tests/fixtures-tracke/package.json +7 -0
- package/tests/fixtures-tracke/tests/auth.test.ts +23 -0
- package/tests/fixtures-tracke/tests/billing.test.ts +14 -0
- package/tests/fixtures-trackf/package.json +5 -0
- package/tests/fixtures-trackf/src/auth.ts +26 -0
- package/tests/fixtures-trackf/src/handlers.ts +35 -0
- package/tests/fixtures-tracki/billing/routes.ts +12 -0
- package/tests/fixtures-tracki/gateway/client.ts +13 -0
- package/tests/git-features.ts +267 -0
- package/tests/init.ts +141 -0
- package/tests/mcp-jit.ts +130 -0
- package/tests/mcp-smoke.ts +191 -0
- package/tests/mcp-trackcd.ts +169 -0
- package/tests/mcp-tracke.ts +229 -0
- package/tests/mcp-trackf.ts +330 -0
- package/tests/mcp-trackg.ts +219 -0
- package/tests/mcp-tracki.ts +174 -0
- package/tests/mcp-watcher.ts +126 -0
- package/tests/optspec.ts +194 -0
- package/tests/parallel-index.ts +333 -0
- package/tests/parallel-read.ts +125 -0
- package/tests/parallel-recovery.ts +241 -0
- package/tests/perf-callers.ts +145 -0
- package/tests/query-parity.ts +184 -0
- package/tests/query-perf.ts +55 -0
- package/tests/scale-parallel-parity.ts +225 -0
- package/tests/scale-test.ts +523 -0
- package/tests/smoke.ts +396 -0
- package/tests/trackcd.ts +325 -0
- package/tests/tracke-collisions.ts +255 -0
- package/tests/tracke.ts +314 -0
- package/tests/trackf-bugs.ts +406 -0
- package/tests/trackf.ts +390 -0
- package/tests/trackg.ts +1372 -0
- package/tests/tracki-boundaries.ts +202 -0
- package/tests/tracki-continuity.ts +253 -0
- package/tests/tracki-contract-diff.ts +249 -0
- package/tests/tracki-external-bundles.ts +341 -0
- package/tests/tracki-preflight.ts +251 -0
- package/tests/verify-roles.ts +51 -0
- package/tests/worker-parity.ts +286 -0
- package/tests/worker-pool.ts +262 -0
- package/tsconfig.json +20 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { Store } from '../db/store.js';
|
|
2
|
+
import { gitChangedFiles, fileDiffHunksSync, isGitRepo } from './git.js';
|
|
3
|
+
import type { SymbolRow } from '../types.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Compute the blast radius of an uncommitted (or between-refs) diff. For each
|
|
7
|
+
* changed file we identify the symbols whose line ranges overlap the diff
|
|
8
|
+
* hunks, then expand by N levels of reverse callers (transitive callers,
|
|
9
|
+
* because they're the code most likely to break).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export interface ChangedSymbol {
|
|
13
|
+
symbol: SymbolRow;
|
|
14
|
+
hunkCount: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface DetectChangesResult {
|
|
18
|
+
fromRef: string | null;
|
|
19
|
+
toRef: string | null;
|
|
20
|
+
changedFiles: Array<{ path: string; hunks: number; symbols: ChangedSymbol[] }>;
|
|
21
|
+
/** Direct changed symbols (the inner symbols in `changedFiles`). */
|
|
22
|
+
directlyChanged: SymbolRow[];
|
|
23
|
+
/** Transitive callers of the directly-changed set (deduped). */
|
|
24
|
+
transitivelyAffected: SymbolRow[];
|
|
25
|
+
elapsedMs: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function detectChanges(
|
|
29
|
+
repoRoot: string, store: Store,
|
|
30
|
+
options: { fromRef?: string; toRef?: string; callerDepth?: number } = {},
|
|
31
|
+
): DetectChangesResult {
|
|
32
|
+
const start = Date.now();
|
|
33
|
+
const callerDepth = options.callerDepth ?? 2;
|
|
34
|
+
const fromRef = options.fromRef ?? null;
|
|
35
|
+
const toRef = options.toRef ?? null;
|
|
36
|
+
if (!isGitRepo(repoRoot)) {
|
|
37
|
+
return { fromRef, toRef, changedFiles: [], directlyChanged: [], transitivelyAffected: [], elapsedMs: Date.now() - start };
|
|
38
|
+
}
|
|
39
|
+
const files = gitChangedFiles(repoRoot, fromRef ?? undefined, toRef ?? undefined);
|
|
40
|
+
if (files.length === 0) {
|
|
41
|
+
return { fromRef, toRef, changedFiles: [], directlyChanged: [], transitivelyAffected: [], elapsedMs: Date.now() - start };
|
|
42
|
+
}
|
|
43
|
+
const dbFiles = new Map(store.listFiles().map(f => [normalize(f.path), f.id]));
|
|
44
|
+
const changedFiles: DetectChangesResult['changedFiles'] = [];
|
|
45
|
+
const directIds = new Set<number>();
|
|
46
|
+
for (const abs of files) {
|
|
47
|
+
const fileId = dbFiles.get(normalize(abs));
|
|
48
|
+
if (fileId === undefined) continue;
|
|
49
|
+
const hunks = fileDiffHunksSync(repoRoot, abs, fromRef ?? undefined, toRef ?? undefined);
|
|
50
|
+
if (hunks.length === 0) continue;
|
|
51
|
+
// Convert 1-indexed git line ranges to 0-indexed Seer line ranges.
|
|
52
|
+
const ranges: Array<[number, number]> = hunks.map(h => [
|
|
53
|
+
Math.max(0, h.newStart - 1),
|
|
54
|
+
Math.max(0, h.newStart - 1 + Math.max(0, h.newLines - 1)),
|
|
55
|
+
]);
|
|
56
|
+
const syms = store.symbolsTouchingLines(fileId, ranges);
|
|
57
|
+
for (const s of syms) directIds.add(s.id);
|
|
58
|
+
changedFiles.push({
|
|
59
|
+
path: abs,
|
|
60
|
+
hunks: hunks.length,
|
|
61
|
+
symbols: syms.map(s => ({ symbol: s, hunkCount: hunks.length })),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
const directly: SymbolRow[] = [];
|
|
65
|
+
for (const id of directIds) {
|
|
66
|
+
const s = store.getSymbolById(id);
|
|
67
|
+
if (s) directly.push(s);
|
|
68
|
+
}
|
|
69
|
+
const transitiveIds = new Set<number>();
|
|
70
|
+
for (const s of directly) {
|
|
71
|
+
for (const id of store.reverseReachable(s.id, callerDepth)) {
|
|
72
|
+
if (!directIds.has(id)) transitiveIds.add(id);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const transitively: SymbolRow[] = [];
|
|
76
|
+
for (const id of transitiveIds) {
|
|
77
|
+
const s = store.getSymbolById(id);
|
|
78
|
+
if (s) transitively.push(s);
|
|
79
|
+
}
|
|
80
|
+
transitively.sort((a, b) => b.pagerank - a.pagerank);
|
|
81
|
+
return {
|
|
82
|
+
fromRef,
|
|
83
|
+
toRef,
|
|
84
|
+
changedFiles,
|
|
85
|
+
directlyChanged: directly,
|
|
86
|
+
transitivelyAffected: transitively,
|
|
87
|
+
elapsedMs: Date.now() - start,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function normalize(p: string): string {
|
|
92
|
+
const n = p.replace(/\\/g, '/');
|
|
93
|
+
return process.platform === 'win32' ? n.toLowerCase() : n;
|
|
94
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import ignore from 'ignore';
|
|
4
|
+
import glob from 'fast-glob';
|
|
5
|
+
|
|
6
|
+
export interface DiscoveredFile {
|
|
7
|
+
absolutePath: string;
|
|
8
|
+
relativePath: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Discovery mode controls how aggressively we filter directories/files before
|
|
13
|
+
* parsing. Modes are layered defaults — finer-grained `includeVendor` /
|
|
14
|
+
* `includeGenerated` toggles can still override the mode's vendor/generated
|
|
15
|
+
* decisions, and `.seerignore` rules apply on top of all of them.
|
|
16
|
+
*
|
|
17
|
+
* - `full` index everything we can parse (only build/meta and .git skipped)
|
|
18
|
+
* - `standard` skip vendor + generated by default (current historical default)
|
|
19
|
+
* - `fast` standard + skip docs/examples/static/assets/migrations — aimed
|
|
20
|
+
* at iterative agent loops where most non-source dirs are dead
|
|
21
|
+
* weight
|
|
22
|
+
*
|
|
23
|
+
* Default is `standard`; `fast` is a deliberate opt-in for power users on
|
|
24
|
+
* very large repos where they want indexing as cheap as possible.
|
|
25
|
+
*/
|
|
26
|
+
export type DiscoveryMode = 'full' | 'standard' | 'fast';
|
|
27
|
+
|
|
28
|
+
export interface DiscoveryOptions {
|
|
29
|
+
/**
|
|
30
|
+
* If true, vendored / generated directories that the default ignore list
|
|
31
|
+
* would skip are included in discovery. Vendored/generated classification
|
|
32
|
+
* still happens at index time — this just lets the user inspect what
|
|
33
|
+
* Seer would normally hide. Off by default; the README and master guide
|
|
34
|
+
* describe this as the "I really do want vendored code" escape hatch.
|
|
35
|
+
*/
|
|
36
|
+
includeVendor?: boolean;
|
|
37
|
+
includeGenerated?: boolean;
|
|
38
|
+
/**
|
|
39
|
+
* Discovery aggressiveness. Defaults to 'standard'. See `DiscoveryMode`.
|
|
40
|
+
*/
|
|
41
|
+
mode?: DiscoveryMode;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Globally-skipped paths that are never source code (build outputs, IDE
|
|
45
|
+
// state, VCS metadata). These are unconditional — `includeVendor` /
|
|
46
|
+
// `includeGenerated` do NOT re-enable them. The user can override by adding
|
|
47
|
+
// a `!pattern` line in `.seerignore`.
|
|
48
|
+
const BUILD_AND_META_IGNORE = [
|
|
49
|
+
'node_modules', '.git', '.hg', '.svn',
|
|
50
|
+
'dist', 'build', 'out', '.next', '.nuxt', '__pycache__',
|
|
51
|
+
'*.min.js', '*.min.css', '*.bundle.js',
|
|
52
|
+
'*.lock', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml',
|
|
53
|
+
// Build outputs across other ecosystems. `target/` is universally Rust
|
|
54
|
+
// (`cargo build`), `obj/` is .NET, `cmake-build-*` is JetBrains/CLion,
|
|
55
|
+
// `_build/` is Erlang/Elixir and some doc generators. Not adding `bin/`
|
|
56
|
+
// here — TypeScript-main and others keep entry scripts there.
|
|
57
|
+
'target/**', '**/target/**',
|
|
58
|
+
'obj/**', '**/obj/**',
|
|
59
|
+
'cmake-build-*/**', '**/cmake-build-*/**',
|
|
60
|
+
'_build/**', '**/_build/**',
|
|
61
|
+
'.gradle/**', '**/.gradle/**',
|
|
62
|
+
'.cache/**',
|
|
63
|
+
'.idea/**', '.vs/**',
|
|
64
|
+
// Unreal-specific build outputs (won't exist in a clean checkout but cheap
|
|
65
|
+
// to add defensively for users who built before indexing).
|
|
66
|
+
'Intermediate/**', '**/Intermediate/**',
|
|
67
|
+
'Saved/**', '**/Saved/**',
|
|
68
|
+
'DerivedDataCache/**',
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
// Vendored dependency roots — discovery-time skip by default, but classified
|
|
72
|
+
// even when included. Match both the top level and any nested depth (Godot
|
|
73
|
+
// uses `thirdparty/`, Unreal uses `Engine/Source/ThirdParty/`, etc.).
|
|
74
|
+
const VENDOR_IGNORE = [
|
|
75
|
+
'vendor/**', '**/vendor/**',
|
|
76
|
+
'vendored/**', '**/vendored/**',
|
|
77
|
+
'Vendored/**', '**/Vendored/**',
|
|
78
|
+
'third_party/**', '**/third_party/**',
|
|
79
|
+
'thirdparty/**', '**/thirdparty/**',
|
|
80
|
+
'ThirdParty/**', '**/ThirdParty/**',
|
|
81
|
+
];
|
|
82
|
+
|
|
83
|
+
// Generated-code patterns that don't earn a place in default indexing. Same
|
|
84
|
+
// "skip-by-default, classify-when-included" model as vendored code.
|
|
85
|
+
const GENERATED_IGNORE = [
|
|
86
|
+
'*.pb.go', '*.pb.ts', '*.pb.h', '*.pb.cc',
|
|
87
|
+
'*.generated.h', '*.gen.cpp', '*.gen.h',
|
|
88
|
+
];
|
|
89
|
+
|
|
90
|
+
// Extra skips active only under `--mode fast`. These directories rarely
|
|
91
|
+
// contain source code that contributes to the call graph — examples are
|
|
92
|
+
// either demos that re-import from src, docs/static/assets are non-code, and
|
|
93
|
+
// migrations are usually flat SQL or schema-only Python. Skipping them in
|
|
94
|
+
// fast mode trims discovery and parse cost meaningfully on large monorepos.
|
|
95
|
+
// Conservative on purpose: anything that might contain real call edges
|
|
96
|
+
// (`tests/`, `src/`, `lib/`) stays indexed even in fast mode.
|
|
97
|
+
const FAST_MODE_EXTRA_IGNORE = [
|
|
98
|
+
'docs/**', '**/docs/**',
|
|
99
|
+
'examples/**', '**/examples/**',
|
|
100
|
+
'example/**', '**/example/**',
|
|
101
|
+
'assets/**', '**/assets/**',
|
|
102
|
+
'static/**', '**/static/**',
|
|
103
|
+
'public/**', '**/public/**',
|
|
104
|
+
'media/**', '**/media/**',
|
|
105
|
+
'fixtures/**', '**/fixtures/**',
|
|
106
|
+
'testdata/**', '**/testdata/**',
|
|
107
|
+
'migrations/**', '**/migrations/**',
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
export async function discoverFiles(repoRoot: string, options: DiscoveryOptions = {}): Promise<DiscoveredFile[]> {
|
|
111
|
+
const absRoot = path.resolve(repoRoot);
|
|
112
|
+
const mode: DiscoveryMode = options.mode ?? 'standard';
|
|
113
|
+
// `full` mode flips both include flags ON regardless of caller intent. We
|
|
114
|
+
// still honor explicit `includeVendor=false` from a caller, but the typical
|
|
115
|
+
// use of `--mode full` is "index literally everything", so the default
|
|
116
|
+
// there is to include them.
|
|
117
|
+
const includeVendor = options.includeVendor ?? (mode === 'full');
|
|
118
|
+
const includeGenerated = options.includeGenerated ?? (mode === 'full');
|
|
119
|
+
|
|
120
|
+
const skip = [...BUILD_AND_META_IGNORE];
|
|
121
|
+
if (!includeVendor) skip.push(...VENDOR_IGNORE);
|
|
122
|
+
if (!includeGenerated) skip.push(...GENERATED_IGNORE);
|
|
123
|
+
if (mode === 'fast') skip.push(...FAST_MODE_EXTRA_IGNORE);
|
|
124
|
+
|
|
125
|
+
// Build ignore rules from .gitignore + optional .seerignore. The two are
|
|
126
|
+
// separate intentionally — .gitignore controls what's committed (often
|
|
127
|
+
// includes build outputs that we ALSO want hidden) while .seerignore is
|
|
128
|
+
// for repo-specific tweaks that don't belong in version control rules
|
|
129
|
+
// (e.g. "don't index our `examples/` folder").
|
|
130
|
+
const ig = ignore();
|
|
131
|
+
const gitignorePath = path.join(absRoot, '.gitignore');
|
|
132
|
+
if (fs.existsSync(gitignorePath)) {
|
|
133
|
+
ig.add(fs.readFileSync(gitignorePath, 'utf8'));
|
|
134
|
+
}
|
|
135
|
+
const seerignorePath = path.join(absRoot, '.seerignore');
|
|
136
|
+
if (fs.existsSync(seerignorePath)) {
|
|
137
|
+
ig.add(fs.readFileSync(seerignorePath, 'utf8'));
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Glob for source files in supported languages
|
|
141
|
+
const entries = await glob(
|
|
142
|
+
[
|
|
143
|
+
'**/*.py', '**/*.pyw',
|
|
144
|
+
'**/*.ts', '**/*.tsx',
|
|
145
|
+
'**/*.js', '**/*.jsx', '**/*.mjs', '**/*.cjs',
|
|
146
|
+
'**/*.go',
|
|
147
|
+
'**/*.java',
|
|
148
|
+
'**/*.rs',
|
|
149
|
+
'**/*.c',
|
|
150
|
+
'**/*.cpp', '**/*.cc', '**/*.cxx', '**/*.c++',
|
|
151
|
+
'**/*.hpp', '**/*.hh', '**/*.h++', '**/*.h',
|
|
152
|
+
'**/*.cs',
|
|
153
|
+
],
|
|
154
|
+
{
|
|
155
|
+
cwd: absRoot,
|
|
156
|
+
ignore: skip,
|
|
157
|
+
onlyFiles: true,
|
|
158
|
+
followSymbolicLinks: false,
|
|
159
|
+
dot: false,
|
|
160
|
+
},
|
|
161
|
+
);
|
|
162
|
+
|
|
163
|
+
// Stable order across runs is a correctness requirement: file IDs are
|
|
164
|
+
// AUTOINCREMENT, and tests/scale invariants depend on the same input
|
|
165
|
+
// producing the same IDs. `fast-glob` on Windows returns files in MFT/
|
|
166
|
+
// FS-cache order, which is not stable run-to-run. Sort here so every
|
|
167
|
+
// downstream stage — the byte semaphore window, parser-worker dispatch,
|
|
168
|
+
// SQLite inserts — sees the same sequence on every invocation.
|
|
169
|
+
return entries
|
|
170
|
+
.filter(rel => !ig.ignores(rel))
|
|
171
|
+
.sort()
|
|
172
|
+
.map(rel => ({
|
|
173
|
+
absolutePath: path.join(absRoot, rel),
|
|
174
|
+
relativePath: rel,
|
|
175
|
+
}));
|
|
176
|
+
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import glob from 'fast-glob';
|
|
4
|
+
import { Store } from '../db/store.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Walk the repo for known package manifests and emit one row per declared
|
|
8
|
+
* dependency. Idempotent: clears `external_dependencies` and re-inserts every
|
|
9
|
+
* call, so deletions in package.json are reflected on the next index.
|
|
10
|
+
*
|
|
11
|
+
* Supported ecosystems:
|
|
12
|
+
* npm package.json / package-lock.json / pnpm-lock.yaml (deps only)
|
|
13
|
+
* cargo Cargo.toml (Cargo.lock not parsed — duplicates would be noisy)
|
|
14
|
+
* pypi requirements.txt, pyproject.toml (PEP 621 [project.dependencies])
|
|
15
|
+
* go go.mod
|
|
16
|
+
*
|
|
17
|
+
* Manifest discovery uses fast-glob with the same ignores as the rest of the
|
|
18
|
+
* indexer (no node_modules, no vendor) so monorepos can be picked up from
|
|
19
|
+
* `packages/foo/package.json`.
|
|
20
|
+
*/
|
|
21
|
+
export async function extractExternalDependencies(repoRoot: string, store: Store): Promise<number> {
|
|
22
|
+
const abs = path.resolve(repoRoot);
|
|
23
|
+
const manifestPatterns = [
|
|
24
|
+
'package.json',
|
|
25
|
+
'**/package.json',
|
|
26
|
+
'Cargo.toml',
|
|
27
|
+
'**/Cargo.toml',
|
|
28
|
+
'pyproject.toml',
|
|
29
|
+
'**/pyproject.toml',
|
|
30
|
+
'requirements*.txt',
|
|
31
|
+
'**/requirements*.txt',
|
|
32
|
+
'go.mod',
|
|
33
|
+
'**/go.mod',
|
|
34
|
+
];
|
|
35
|
+
const ignored = [
|
|
36
|
+
'node_modules/**', '**/node_modules/**',
|
|
37
|
+
'vendor/**', '**/vendor/**', 'vendored/**', '**/vendored/**',
|
|
38
|
+
'third_party/**', '**/third_party/**', 'thirdparty/**', '**/thirdparty/**',
|
|
39
|
+
'target/**', '**/target/**',
|
|
40
|
+
'dist/**', '**/dist/**',
|
|
41
|
+
'.git/**',
|
|
42
|
+
];
|
|
43
|
+
const matches = await glob(manifestPatterns, {
|
|
44
|
+
cwd: abs, ignore: ignored, onlyFiles: true, followSymbolicLinks: false, dot: false,
|
|
45
|
+
unique: true,
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
store.clearExternalDeps();
|
|
49
|
+
let inserted = 0;
|
|
50
|
+
for (const rel of matches) {
|
|
51
|
+
const filePath = path.join(abs, rel);
|
|
52
|
+
let content: string;
|
|
53
|
+
try { content = fs.readFileSync(filePath, 'utf8'); } catch { continue; }
|
|
54
|
+
if (rel.endsWith('package.json') || rel === 'package.json') {
|
|
55
|
+
inserted += parsePackageJson(content, rel, store);
|
|
56
|
+
} else if (rel.endsWith('Cargo.toml') || rel === 'Cargo.toml') {
|
|
57
|
+
inserted += parseCargoToml(content, rel, store);
|
|
58
|
+
} else if (rel.endsWith('pyproject.toml') || rel === 'pyproject.toml') {
|
|
59
|
+
inserted += parsePyproject(content, rel, store);
|
|
60
|
+
} else if (rel.endsWith('.txt')) {
|
|
61
|
+
inserted += parseRequirementsTxt(content, rel, store);
|
|
62
|
+
} else if (rel.endsWith('go.mod') || rel === 'go.mod') {
|
|
63
|
+
inserted += parseGoMod(content, rel, store);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return inserted;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function parsePackageJson(content: string, manifestPath: string, store: Store): number {
|
|
70
|
+
let json: any;
|
|
71
|
+
try { json = JSON.parse(content); } catch { return 0; }
|
|
72
|
+
if (!json || typeof json !== 'object') return 0;
|
|
73
|
+
let count = 0;
|
|
74
|
+
const groups: Array<[Record<string, unknown> | undefined, 0 | 1]> = [
|
|
75
|
+
[json.dependencies, 0],
|
|
76
|
+
[json.devDependencies, 1],
|
|
77
|
+
[json.peerDependencies, 0],
|
|
78
|
+
[json.optionalDependencies, 0],
|
|
79
|
+
];
|
|
80
|
+
for (const [group, isDev] of groups) {
|
|
81
|
+
if (!group || typeof group !== 'object') continue;
|
|
82
|
+
for (const [name, version] of Object.entries(group)) {
|
|
83
|
+
if (!name) continue;
|
|
84
|
+
store.insertExternalDep('npm', name, typeof version === 'string' ? version : null, manifestPath, isDev);
|
|
85
|
+
count++;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return count;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function parseCargoToml(content: string, manifestPath: string, store: Store): number {
|
|
92
|
+
// Lightweight TOML parser for [dependencies] / [dev-dependencies] sections.
|
|
93
|
+
// We intentionally don't pull in a full TOML lib — Cargo manifests are
|
|
94
|
+
// well-formed enough that a section-aware line walk suffices.
|
|
95
|
+
let count = 0;
|
|
96
|
+
let section = '';
|
|
97
|
+
for (const rawLine of content.split('\n')) {
|
|
98
|
+
const line = rawLine.replace(/#.*$/, '').trim();
|
|
99
|
+
if (!line) continue;
|
|
100
|
+
const sectionMatch = line.match(/^\[(.+?)\]$/);
|
|
101
|
+
if (sectionMatch) {
|
|
102
|
+
section = sectionMatch[1].trim();
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const inDeps = section === 'dependencies' || section === 'dev-dependencies' || section === 'build-dependencies';
|
|
106
|
+
if (!inDeps) continue;
|
|
107
|
+
const eq = line.indexOf('=');
|
|
108
|
+
if (eq < 0) continue;
|
|
109
|
+
const name = line.slice(0, eq).trim();
|
|
110
|
+
const rest = line.slice(eq + 1).trim();
|
|
111
|
+
if (!name || /[\[{]/.test(name)) continue;
|
|
112
|
+
let version: string | null = null;
|
|
113
|
+
const strMatch = rest.match(/^"([^"]+)"/);
|
|
114
|
+
if (strMatch) version = strMatch[1];
|
|
115
|
+
else {
|
|
116
|
+
const inlineVer = rest.match(/version\s*=\s*"([^"]+)"/);
|
|
117
|
+
if (inlineVer) version = inlineVer[1];
|
|
118
|
+
}
|
|
119
|
+
const isDev: 0 | 1 = section === 'dev-dependencies' ? 1 : 0;
|
|
120
|
+
store.insertExternalDep('cargo', name, version, manifestPath, isDev);
|
|
121
|
+
count++;
|
|
122
|
+
}
|
|
123
|
+
return count;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function parsePyproject(content: string, manifestPath: string, store: Store): number {
|
|
127
|
+
// [project] dependencies / optional-dependencies — Poetry uses
|
|
128
|
+
// [tool.poetry.dependencies] in addition. We handle both.
|
|
129
|
+
let count = 0;
|
|
130
|
+
let section = '';
|
|
131
|
+
let inList = false;
|
|
132
|
+
let buf: string[] = [];
|
|
133
|
+
const flushList = (isDev: 0 | 1): void => {
|
|
134
|
+
for (const item of buf) {
|
|
135
|
+
const m = item.match(/^"([^"]+)"|^'([^']+)'/);
|
|
136
|
+
if (!m) continue;
|
|
137
|
+
const spec = (m[1] ?? m[2] ?? '').trim();
|
|
138
|
+
if (!spec) continue;
|
|
139
|
+
// Strip extras / version: "package[extras]>=1.2.3"
|
|
140
|
+
const nameMatch = spec.match(/^([A-Za-z0-9_.\-]+)/);
|
|
141
|
+
if (!nameMatch) continue;
|
|
142
|
+
const name = nameMatch[1];
|
|
143
|
+
const versionMatch = spec.slice(nameMatch[0].length).match(/[<>=~!^].+/);
|
|
144
|
+
const version = versionMatch ? versionMatch[0].trim() : null;
|
|
145
|
+
store.insertExternalDep('pypi', name, version, manifestPath, isDev);
|
|
146
|
+
count++;
|
|
147
|
+
}
|
|
148
|
+
buf = [];
|
|
149
|
+
};
|
|
150
|
+
const lines = content.split('\n');
|
|
151
|
+
for (let i = 0; i < lines.length; i++) {
|
|
152
|
+
const raw = lines[i];
|
|
153
|
+
const line = raw.replace(/#.*$/, '').trim();
|
|
154
|
+
const sectionMatch = line.match(/^\[(.+?)\]$/);
|
|
155
|
+
if (sectionMatch) {
|
|
156
|
+
if (inList) { flushList(section.includes('dev') ? 1 : 0); inList = false; }
|
|
157
|
+
section = sectionMatch[1].trim();
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (section === 'project' || section === 'tool.poetry.dependencies' || section === 'tool.poetry.dev-dependencies') {
|
|
161
|
+
// [project] dependencies = ["pkg>=1.0", ...]
|
|
162
|
+
if (section === 'project' && /^dependencies\s*=\s*\[/.test(line)) {
|
|
163
|
+
inList = true;
|
|
164
|
+
const after = line.replace(/^dependencies\s*=\s*\[/, '');
|
|
165
|
+
if (after.includes(']')) {
|
|
166
|
+
for (const item of after.replace(/\].*$/, '').split(',')) buf.push(item.trim());
|
|
167
|
+
flushList(0);
|
|
168
|
+
inList = false;
|
|
169
|
+
} else {
|
|
170
|
+
buf.push(after);
|
|
171
|
+
}
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
if (inList) {
|
|
175
|
+
if (line.includes(']')) {
|
|
176
|
+
for (const item of line.replace(/\].*$/, '').split(',')) if (item.trim()) buf.push(item.trim());
|
|
177
|
+
flushList(0);
|
|
178
|
+
inList = false;
|
|
179
|
+
} else {
|
|
180
|
+
for (const item of line.split(',')) if (item.trim()) buf.push(item.trim());
|
|
181
|
+
}
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
if (section.startsWith('tool.poetry') && /^[A-Za-z0-9_.\-]+\s*=/.test(line)) {
|
|
185
|
+
const eq = line.indexOf('=');
|
|
186
|
+
const name = line.slice(0, eq).trim();
|
|
187
|
+
const rest = line.slice(eq + 1).trim();
|
|
188
|
+
const verMatch = rest.match(/^"([^"]+)"/);
|
|
189
|
+
const version = verMatch ? verMatch[1] : null;
|
|
190
|
+
const isDev: 0 | 1 = section.includes('dev') ? 1 : 0;
|
|
191
|
+
store.insertExternalDep('pypi', name, version, manifestPath, isDev);
|
|
192
|
+
count++;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (inList) flushList(section.includes('dev') ? 1 : 0);
|
|
197
|
+
return count;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function parseRequirementsTxt(content: string, manifestPath: string, store: Store): number {
|
|
201
|
+
let count = 0;
|
|
202
|
+
const isDev: 0 | 1 = /dev|test/i.test(manifestPath) ? 1 : 0;
|
|
203
|
+
for (const raw of content.split('\n')) {
|
|
204
|
+
const line = raw.replace(/#.*$/, '').trim();
|
|
205
|
+
if (!line || line.startsWith('-')) continue;
|
|
206
|
+
const m = line.match(/^([A-Za-z0-9_.\-]+)/);
|
|
207
|
+
if (!m) continue;
|
|
208
|
+
const name = m[1];
|
|
209
|
+
const rest = line.slice(m[0].length);
|
|
210
|
+
const ver = rest.match(/[<>=~!^].+/);
|
|
211
|
+
store.insertExternalDep('pypi', name, ver ? ver[0].trim() : null, manifestPath, isDev);
|
|
212
|
+
count++;
|
|
213
|
+
}
|
|
214
|
+
return count;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function parseGoMod(content: string, manifestPath: string, store: Store): number {
|
|
218
|
+
let count = 0;
|
|
219
|
+
let inRequire = false;
|
|
220
|
+
for (const raw of content.split('\n')) {
|
|
221
|
+
const line = raw.replace(/\/\/.*$/, '').trim();
|
|
222
|
+
if (!line) continue;
|
|
223
|
+
if (line.startsWith('require (')) { inRequire = true; continue; }
|
|
224
|
+
if (line === ')') { inRequire = false; continue; }
|
|
225
|
+
if (line.startsWith('require ')) {
|
|
226
|
+
// single-line: `require mod v1.2.3`
|
|
227
|
+
const parts = line.slice('require '.length).trim().split(/\s+/);
|
|
228
|
+
if (parts.length >= 2) {
|
|
229
|
+
store.insertExternalDep('go', parts[0], parts[1], manifestPath, 0);
|
|
230
|
+
count++;
|
|
231
|
+
}
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
if (inRequire) {
|
|
235
|
+
const parts = line.split(/\s+/);
|
|
236
|
+
if (parts.length >= 2 && !/^\/\//.test(parts[0])) {
|
|
237
|
+
store.insertExternalDep('go', parts[0], parts[1], manifestPath, 0);
|
|
238
|
+
count++;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return count;
|
|
243
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import crypto from 'crypto';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { Indexer } from './index.js';
|
|
5
|
+
import { Store } from '../db/store.js';
|
|
6
|
+
import { discoverFiles } from './discovery.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Quick freshness check + targeted re-index for MCP/CLI queries.
|
|
10
|
+
*
|
|
11
|
+
* Design contract: the watcher (when running) keeps the index warm by marking
|
|
12
|
+
* files dirty in the background. JIT sync runs before every query as the
|
|
13
|
+
* correctness layer — it does the actual reindex of dirty files so a query
|
|
14
|
+
* returning right now reflects the current workspace state.
|
|
15
|
+
*
|
|
16
|
+
* Implementation: discover the workspace, compare on-disk content hashes
|
|
17
|
+
* against what the DB says, and only reindex files whose hash changed.
|
|
18
|
+
* Unchanged files are skipped entirely; we don't touch the parser for them.
|
|
19
|
+
*
|
|
20
|
+
* The intentional bias here is correctness over latency: we re-discover the
|
|
21
|
+
* full workspace each time so newly-added or newly-renamed files are not
|
|
22
|
+
* missed. For very large repos this is still cheap (a glob + dir walk
|
|
23
|
+
* over 10-100k files is sub-second) compared to a single tree-sitter parse,
|
|
24
|
+
* and you only do it once per JIT call.
|
|
25
|
+
*/
|
|
26
|
+
export interface FreshnessReport {
|
|
27
|
+
/** Files where the on-disk hash differed from the DB. Reindexed. */
|
|
28
|
+
dirtyReindexed: number;
|
|
29
|
+
/** Files that vanished from disk since the last index. Pruned. */
|
|
30
|
+
removed: number;
|
|
31
|
+
/** Files newly seen this run (not in the DB yet). Indexed. */
|
|
32
|
+
added: number;
|
|
33
|
+
/** Total wall time in ms. */
|
|
34
|
+
elapsedMs: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Cheap content hash matching what the indexer uses internally. Kept here as
|
|
39
|
+
* a duplicate (not exported from indexer/index.ts) because the indexer's
|
|
40
|
+
* version is in the hot loop and we don't want to widen its export surface.
|
|
41
|
+
*/
|
|
42
|
+
function sha256Short(content: string): string {
|
|
43
|
+
return crypto.createHash('sha256').update(content, 'utf8').digest('hex').slice(0, 16);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Inspect a workspace for changes since the last index, then reindex only
|
|
48
|
+
* the files that need it. Designed to run before every MCP query.
|
|
49
|
+
*
|
|
50
|
+
* @param store a writable Store (NOT readonly — we may need to mutate)
|
|
51
|
+
* @param indexer an Indexer over the same store
|
|
52
|
+
* @param repoRoot the workspace path used at index time
|
|
53
|
+
* @param options.maxDirty cap reindex work per call. When the dirty set is
|
|
54
|
+
* larger than this, we still run a full `indexer.indexDirectory()` because
|
|
55
|
+
* a partial JIT pass would leave the index inconsistent (resolveEdges
|
|
56
|
+
* needs the full graph). Defaults to 200 — small enough to keep the
|
|
57
|
+
* "type a few characters and ask" workflow snappy.
|
|
58
|
+
*/
|
|
59
|
+
export async function jitSync(
|
|
60
|
+
store: Store,
|
|
61
|
+
indexer: Indexer,
|
|
62
|
+
repoRoot: string,
|
|
63
|
+
options: { maxDirty?: number; verbose?: boolean } = {},
|
|
64
|
+
): Promise<FreshnessReport> {
|
|
65
|
+
const start = Date.now();
|
|
66
|
+
const maxDirty = options.maxDirty ?? 200;
|
|
67
|
+
|
|
68
|
+
const absRoot = path.resolve(repoRoot);
|
|
69
|
+
|
|
70
|
+
// 1. Snapshot what the DB knows.
|
|
71
|
+
const dbFiles = store.listFiles();
|
|
72
|
+
const dbByPath = new Map(dbFiles.map(f => [normalizeForCompare(f.path), f]));
|
|
73
|
+
|
|
74
|
+
// 2. Walk the workspace and find candidate files. discoverFiles() applies
|
|
75
|
+
// the same ignore rules the indexer uses, so freshness can't be
|
|
76
|
+
// misled by build artifacts or `vendor/` entries.
|
|
77
|
+
const discovered = await discoverFiles(absRoot);
|
|
78
|
+
const discoveredByPath = new Map<string, string>();
|
|
79
|
+
for (const d of discovered) {
|
|
80
|
+
discoveredByPath.set(normalizeForCompare(d.absolutePath), d.relativePath);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// 3. Identify added / removed / candidate-dirty files.
|
|
84
|
+
const added: string[] = [];
|
|
85
|
+
const removed: number[] = [];
|
|
86
|
+
const candidateDirty: typeof dbFiles = [];
|
|
87
|
+
for (const f of dbFiles) {
|
|
88
|
+
const key = normalizeForCompare(f.path);
|
|
89
|
+
if (!discoveredByPath.has(key)) {
|
|
90
|
+
removed.push(f.id);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
candidateDirty.push(f);
|
|
94
|
+
}
|
|
95
|
+
for (const [key, _rel] of discoveredByPath) {
|
|
96
|
+
if (!dbByPath.has(key)) added.push(key);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// 4. Hash each candidate. Same trade-off as the indexer: read everything
|
|
100
|
+
// we'd parse anyway, but if the hash matches we never spend time on
|
|
101
|
+
// the parser. We stop early as soon as we cross `maxDirty` so a giant
|
|
102
|
+
// change like a git checkout falls back to a full reindex.
|
|
103
|
+
const dirty: string[] = [];
|
|
104
|
+
for (const f of candidateDirty) {
|
|
105
|
+
if (dirty.length + added.length >= maxDirty) break;
|
|
106
|
+
let content: string;
|
|
107
|
+
try {
|
|
108
|
+
content = await fs.promises.readFile(f.path, 'utf8');
|
|
109
|
+
} catch {
|
|
110
|
+
// File became unreadable mid-check (rename, permission flip). Treat
|
|
111
|
+
// as removed so the next pass cleans it up.
|
|
112
|
+
removed.push(f.id);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const hash = sha256Short(content);
|
|
116
|
+
if (hash !== f.hash) dirty.push(f.path);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const fullReindexNeeded =
|
|
120
|
+
dirty.length + added.length >= maxDirty || removed.length > 0;
|
|
121
|
+
|
|
122
|
+
if (dirty.length === 0 && added.length === 0 && removed.length === 0) {
|
|
123
|
+
return { dirtyReindexed: 0, removed: 0, added: 0, elapsedMs: Date.now() - start };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (fullReindexNeeded) {
|
|
127
|
+
// Cheaper to invoke the full pipeline than to surgically remove files
|
|
128
|
+
// and reconcile edge graphs. The indexer's cache means unchanged files
|
|
129
|
+
// are still skipped at parse time, so this is O(dirty + added + |touched|)
|
|
130
|
+
// not O(|workspace|).
|
|
131
|
+
// JIT pins parallel:false. The dirty set is small (≤ maxDirty=200) and
|
|
132
|
+
// worker spawn cost dominates the wins at this scale; serial is the
|
|
133
|
+
// right default for the snappy "edit + ask" loop. MCP servers that want
|
|
134
|
+
// parallel JIT can override later via an option.
|
|
135
|
+
const result = await indexer.indexDirectory(absRoot, { quiet: !options.verbose, parallel: false });
|
|
136
|
+
return {
|
|
137
|
+
dirtyReindexed: dirty.length,
|
|
138
|
+
removed: removed.length,
|
|
139
|
+
added: added.length,
|
|
140
|
+
elapsedMs: Date.now() - start + (result.elapsedMs ?? 0),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Targeted JIT path: dirty/added files only, no full re-discovery. Reuse
|
|
145
|
+
// the indexer's machinery by calling indexDirectory — its cache skips
|
|
146
|
+
// unchanged files. This is dominated by the discovery walk we already did,
|
|
147
|
+
// so the marginal cost is small.
|
|
148
|
+
await indexer.indexDirectory(absRoot, { quiet: !options.verbose, parallel: false });
|
|
149
|
+
return {
|
|
150
|
+
dirtyReindexed: dirty.length,
|
|
151
|
+
removed: removed.length,
|
|
152
|
+
added: added.length,
|
|
153
|
+
elapsedMs: Date.now() - start,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Normalize a path for comparison across the OS-specific quirks we hit on
|
|
159
|
+
* Windows. We index with backslashes; discovery resolves through `path.join`
|
|
160
|
+
* which also produces backslashes. Read-only callers might pass a slash-form
|
|
161
|
+
* path through the MCP layer; lowercase folds Windows case-insensitivity.
|
|
162
|
+
*/
|
|
163
|
+
function normalizeForCompare(p: string): string {
|
|
164
|
+
const norm = p.replace(/\\/g, '/');
|
|
165
|
+
return process.platform === 'win32' ? norm.toLowerCase() : norm;
|
|
166
|
+
}
|