septum 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -3
- package/package.json +5 -5
- package/src/cli/commands/check.ts +32 -122
- package/src/cli/commands/hook.ts +45 -12
- package/src/cli/commands/init.ts +6 -4
- package/src/cli/commands/query.ts +45 -5
- package/src/cli/commands/slice.ts +4 -2
- package/src/cli/index.ts +17 -6
- package/src/cli/templates/hooks.ts +6 -6
- package/src/cli/templates/rules.ts +35 -23
- package/src/cli/templates/skill.ts +47 -51
- package/src/core/boundary/evaluator.ts +217 -24
- package/src/core/boundary/staged-auditor.ts +47 -0
- package/src/core/config/loader.ts +29 -12
- package/src/core/database/repositories/dependency.repository.ts +252 -66
- package/src/core/database/repositories/domain.repository.ts +96 -39
- package/src/core/database/repositories/file.repository.ts +20 -19
- package/src/core/database/repositories/symbol.repository.ts +115 -5
- package/src/core/database/repository.ts +19 -2
- package/src/core/database/schema.ts +3 -2
- package/src/core/ingestion/pipeline.ts +96 -60
- package/src/core/parser/boundary-tracker.ts +145 -0
- package/src/core/parser/extractors/frontend.ts +44 -90
- package/src/core/parser/extractors/go.ts +200 -21
- package/src/core/parser/extractors/laravel-semantic.ts +18 -12
- package/src/core/parser/extractors/mcp-cli-semantic.ts +173 -44
- package/src/core/parser/extractors/nestjs-semantic.ts +20 -5
- package/src/core/parser/extractors/php.ts +293 -31
- package/src/core/parser/extractors/python.ts +242 -10
- package/src/core/parser/extractors/typescript.ts +604 -117
- package/src/core/parser/tree-sitter.ts +6 -1
- package/src/core/parser/wasm-assets.ts +17 -0
- package/src/core/parser/wasm-loader.ts +274 -0
- package/src/core/resolver/call-graph-tracer.ts +49 -17
- package/src/core/resolver/module-resolver.ts +88 -26
- package/src/core/resolver/vertical-slice-tracer.ts +34 -11
- package/src/core/session/session-manager.ts +55 -39
- package/src/core/topology/environment-detector.ts +109 -6
- package/src/mcp/schemas.ts +146 -29
- package/src/mcp/server.ts +177 -114
- package/src/mcp/tools/check-boundary.ts +159 -41
- package/src/mcp/tools/get-domain-catalog.ts +30 -15
- package/src/mcp/tools/get-environment-topology.ts +3 -6
- package/src/mcp/tools/get-feature-context.ts +35 -20
- package/src/mcp/tools/get-symbol-hotspots.ts +4 -10
- package/src/mcp/tools/locate-symbol.ts +37 -16
- package/src/mcp/tools/register-domain.ts +3 -12
- package/src/mcp/tools/register-feature.ts +25 -32
- package/src/mcp/tools/trace-vertical-slice.ts +8 -8
- package/src/types/index.ts +56 -6
- package/src/types/wasm.d.ts +4 -0
|
@@ -44,25 +44,22 @@ export class FileRepository {
|
|
|
44
44
|
mtimeMs: number = 0,
|
|
45
45
|
sizeBytes: number = 0
|
|
46
46
|
): number {
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
.run(domainId, path, archetype, contentHash, lineCount, mtimeMs, sizeBytes);
|
|
64
|
-
return Number(result.lastInsertRowid);
|
|
65
|
-
}
|
|
47
|
+
const row = this.db
|
|
48
|
+
.query<{ id: number }, [number, string, string, string, number, number, number]>(
|
|
49
|
+
`INSERT INTO files (domain_id, path, archetype, content_hash, line_count, mtime_ms, size_bytes)
|
|
50
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
51
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
52
|
+
domain_id = excluded.domain_id,
|
|
53
|
+
archetype = excluded.archetype,
|
|
54
|
+
content_hash = excluded.content_hash,
|
|
55
|
+
line_count = excluded.line_count,
|
|
56
|
+
mtime_ms = excluded.mtime_ms,
|
|
57
|
+
size_bytes = excluded.size_bytes,
|
|
58
|
+
last_scanned_at = CURRENT_TIMESTAMP
|
|
59
|
+
RETURNING id`
|
|
60
|
+
)
|
|
61
|
+
.get(domainId, path, archetype, contentHash, lineCount, mtimeMs, sizeBytes);
|
|
62
|
+
return row ? Number(row.id) : 0;
|
|
66
63
|
}
|
|
67
64
|
|
|
68
65
|
public updateFileMetadataOnly(id: number, mtimeMs: number, sizeBytes: number): void {
|
|
@@ -100,6 +97,10 @@ export class FileRepository {
|
|
|
100
97
|
this.db.query("DELETE FROM files WHERE path = ?").run(path);
|
|
101
98
|
}
|
|
102
99
|
|
|
100
|
+
public deleteFileByPath(path: string): void {
|
|
101
|
+
this.deleteFile(path);
|
|
102
|
+
}
|
|
103
|
+
|
|
103
104
|
public getFileWithDomain(
|
|
104
105
|
filePath: string
|
|
105
106
|
): { file: FileRecord; domain: DomainRecord } | null {
|
|
@@ -5,9 +5,31 @@ import type {
|
|
|
5
5
|
ExtractedSymbol,
|
|
6
6
|
FileRecord,
|
|
7
7
|
GetSymbolDetailsResponse,
|
|
8
|
+
ResolvedOutboundCall,
|
|
9
|
+
SymbolComplexitySmell,
|
|
8
10
|
SymbolKind,
|
|
9
11
|
SymbolRecord,
|
|
10
12
|
} from "../../../types/index.ts";
|
|
13
|
+
import { DependencyRepository } from "./dependency.repository.ts";
|
|
14
|
+
|
|
15
|
+
const SQL_KEYWORD_BLACKLIST = new Set([
|
|
16
|
+
"VALUES",
|
|
17
|
+
"WHERE",
|
|
18
|
+
"SELECT",
|
|
19
|
+
"FROM",
|
|
20
|
+
"JOIN",
|
|
21
|
+
"LIKE",
|
|
22
|
+
"INSERT",
|
|
23
|
+
"UPDATE",
|
|
24
|
+
"DELETE",
|
|
25
|
+
]);
|
|
26
|
+
|
|
27
|
+
function stripCommentsAndStrings(source: string): string {
|
|
28
|
+
return source
|
|
29
|
+
.replace(/\/\*[\s\S]*?\*\//g, " ")
|
|
30
|
+
.replace(/\/\/.*$/gm, " ")
|
|
31
|
+
.replace(/'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"|`(?:\\.|[^`\\])*`/g, '""');
|
|
32
|
+
}
|
|
11
33
|
|
|
12
34
|
export class SymbolRepository {
|
|
13
35
|
constructor(private db: Database) {}
|
|
@@ -354,6 +376,10 @@ export class SymbolRepository {
|
|
|
354
376
|
siblingMethods = classMethods.map((s) => s.name.replace(`${row.name}::`, ""));
|
|
355
377
|
}
|
|
356
378
|
|
|
379
|
+
siblingMethods = Array.from(new Set(siblingMethods)).filter(
|
|
380
|
+
(m) => !SQL_KEYWORD_BLACKLIST.has(m.toUpperCase())
|
|
381
|
+
);
|
|
382
|
+
|
|
357
383
|
// 2. Dependencies Injected (from signature type-hints and constructor)
|
|
358
384
|
const dependenciesInjected = new Set<string>();
|
|
359
385
|
const extractTypeHints = (sig: string) => {
|
|
@@ -398,30 +424,111 @@ export class SymbolRepository {
|
|
|
398
424
|
const lines = fileContent.split(/\r?\n/);
|
|
399
425
|
const start = Math.max(0, row.line_start - 1);
|
|
400
426
|
const end = Math.min(lines.length, row.line_end);
|
|
401
|
-
const
|
|
427
|
+
const rawSlice = lines.slice(start, end).join("\n");
|
|
428
|
+
const sanitizedSlice = stripCommentsAndStrings(rawSlice);
|
|
402
429
|
|
|
403
430
|
// Match Static invocations: ClassName::methodName(
|
|
404
|
-
const staticCalls =
|
|
431
|
+
const staticCalls = sanitizedSlice.matchAll(/([A-Z][a-zA-Z0-9_]+)::([a-zA-Z0-9_]+)\s*\(/g);
|
|
405
432
|
for (const m of staticCalls) {
|
|
406
433
|
if (m[1] !== "self" && m[1] !== "parent" && m[1] !== "static") {
|
|
407
434
|
outboundCalls.add(`${m[1]}::${m[2]}`);
|
|
408
435
|
}
|
|
409
436
|
}
|
|
410
437
|
|
|
411
|
-
// Match property calls: $this->property->method(
|
|
412
|
-
const propCalls =
|
|
438
|
+
// Match PHP property calls: $this->property->method(
|
|
439
|
+
const propCalls = sanitizedSlice.matchAll(/\$this->([a-zA-Z0-9_]+)->([a-zA-Z0-9_]+)\s*\(/g);
|
|
413
440
|
for (const m of propCalls) {
|
|
414
|
-
// Capitalize property name for clean domain inference (e.g. orderService -> OrderService::cancelOrder)
|
|
415
441
|
const propName = m[1];
|
|
416
442
|
const methodName = m[2];
|
|
417
443
|
const inferredClass = propName.charAt(0).toUpperCase() + propName.slice(1);
|
|
418
444
|
outboundCalls.add(`${inferredClass}::${methodName}`);
|
|
419
445
|
}
|
|
446
|
+
|
|
447
|
+
// Match TS/JS property calls: this.property.method(
|
|
448
|
+
const tsPropCalls = sanitizedSlice.matchAll(/\bthis\.([a-zA-Z0-9_]+)\.([a-zA-Z0-9_]+)\s*\(/g);
|
|
449
|
+
for (const m of tsPropCalls) {
|
|
450
|
+
const propName = m[1];
|
|
451
|
+
const methodName = m[2];
|
|
452
|
+
if (propName !== "db") {
|
|
453
|
+
const inferredClass = propName.charAt(0).toUpperCase() + propName.slice(1);
|
|
454
|
+
outboundCalls.add(`${inferredClass}::${methodName}`);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
420
457
|
}
|
|
421
458
|
} catch {
|
|
422
459
|
// Graceful fallback if file not physically accessible
|
|
423
460
|
}
|
|
424
461
|
|
|
462
|
+
const depRepo = new DependencyRepository(this.db);
|
|
463
|
+
const connectedFiles = depRepo.getFileConnections(row.file_path);
|
|
464
|
+
|
|
465
|
+
// 4. Resolve Outbound Calls to target symbol line ranges
|
|
466
|
+
const resolvedOutboundCalls: ResolvedOutboundCall[] = [];
|
|
467
|
+
const lookupStmt = this.db.query<
|
|
468
|
+
{
|
|
469
|
+
name: string;
|
|
470
|
+
line_start: number;
|
|
471
|
+
line_end: number;
|
|
472
|
+
line_count: number;
|
|
473
|
+
file_path: string;
|
|
474
|
+
domain_name: string;
|
|
475
|
+
},
|
|
476
|
+
[string, string, string, string]
|
|
477
|
+
>(`
|
|
478
|
+
SELECT
|
|
479
|
+
s.name,
|
|
480
|
+
s.line_start,
|
|
481
|
+
s.line_end,
|
|
482
|
+
s.line_count,
|
|
483
|
+
f.path AS file_path,
|
|
484
|
+
d.name AS domain_name
|
|
485
|
+
FROM symbols s
|
|
486
|
+
JOIN files f ON s.file_id = f.id
|
|
487
|
+
JOIN domains d ON f.domain_id = d.id
|
|
488
|
+
WHERE (s.name = ? OR s.name LIKE ? OR s.name LIKE ?)
|
|
489
|
+
ORDER BY CASE WHEN s.name = ? THEN 1 ELSE 2 END, s.line_start ASC
|
|
490
|
+
LIMIT 1
|
|
491
|
+
`);
|
|
492
|
+
|
|
493
|
+
for (const call of outboundCalls) {
|
|
494
|
+
try {
|
|
495
|
+
const target = lookupStmt.get(call, `%::${call}`, `${call}::%`, call);
|
|
496
|
+
if (target) {
|
|
497
|
+
resolvedOutboundCalls.push({
|
|
498
|
+
symbol: call,
|
|
499
|
+
target_file: target.file_path,
|
|
500
|
+
target_range: {
|
|
501
|
+
start_line: target.line_start,
|
|
502
|
+
end_line: target.line_end,
|
|
503
|
+
total_lines: target.line_count,
|
|
504
|
+
},
|
|
505
|
+
domain: target.domain_name,
|
|
506
|
+
is_resolved: true,
|
|
507
|
+
});
|
|
508
|
+
} else {
|
|
509
|
+
resolvedOutboundCalls.push({
|
|
510
|
+
symbol: call,
|
|
511
|
+
is_resolved: false,
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
} catch {
|
|
515
|
+
resolvedOutboundCalls.push({
|
|
516
|
+
symbol: call,
|
|
517
|
+
is_resolved: false,
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const complexitySmell: SymbolComplexitySmell | undefined =
|
|
523
|
+
row.line_count > 300
|
|
524
|
+
? {
|
|
525
|
+
is_high_complexity: true,
|
|
526
|
+
threshold: 300,
|
|
527
|
+
total_lines: row.line_count,
|
|
528
|
+
warning: `Symbol '${row.name}' spans ${row.line_count} lines (exceeds 300-line threshold). Candidate for decomposition/refactoring.`,
|
|
529
|
+
}
|
|
530
|
+
: undefined;
|
|
531
|
+
|
|
425
532
|
return {
|
|
426
533
|
symbol: row.name,
|
|
427
534
|
kind: row.kind,
|
|
@@ -436,7 +543,10 @@ export class SymbolRepository {
|
|
|
436
543
|
visibility: row.visibility,
|
|
437
544
|
dependencies_injected: Array.from(dependenciesInjected),
|
|
438
545
|
outbound_calls: Array.from(outboundCalls),
|
|
546
|
+
resolved_outbound_calls: resolvedOutboundCalls.length > 0 ? resolvedOutboundCalls : undefined,
|
|
547
|
+
complexity_smell: complexitySmell,
|
|
439
548
|
sibling_methods: siblingMethods.length > 0 ? siblingMethods : undefined,
|
|
549
|
+
connected_files: connectedFiles,
|
|
440
550
|
};
|
|
441
551
|
}
|
|
442
552
|
|
|
@@ -122,11 +122,16 @@ export class SeptumRepository {
|
|
|
122
122
|
this.files.deleteFile(path);
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
public deleteFileByPath(path: string): void {
|
|
126
|
+
this.files.deleteFileByPath(path);
|
|
127
|
+
}
|
|
128
|
+
|
|
125
129
|
public getDomainCatalog(
|
|
126
130
|
domainName: string,
|
|
127
|
-
archetypeFilter?: string
|
|
131
|
+
archetypeFilter?: string,
|
|
132
|
+
options?: { detailLevel?: "compact" | "full"; fileFilter?: string }
|
|
128
133
|
): DomainCatalogResponse | null {
|
|
129
|
-
return this.domains.getDomainCatalog(domainName, archetypeFilter);
|
|
134
|
+
return this.domains.getDomainCatalog(domainName, archetypeFilter, options);
|
|
130
135
|
}
|
|
131
136
|
|
|
132
137
|
// --- Symbol Methods ---
|
|
@@ -171,10 +176,22 @@ export class SeptumRepository {
|
|
|
171
176
|
return this.dependencies.getAllVerticalSlices();
|
|
172
177
|
}
|
|
173
178
|
|
|
179
|
+
public countVerticalSlices(): number {
|
|
180
|
+
return this.dependencies.countVerticalSlices();
|
|
181
|
+
}
|
|
182
|
+
|
|
174
183
|
public clearVerticalSlices(domainId?: number): void {
|
|
175
184
|
this.dependencies.clearVerticalSlices(domainId);
|
|
176
185
|
}
|
|
177
186
|
|
|
187
|
+
public getDomainFileConnections(domainId?: number) {
|
|
188
|
+
return this.dependencies.getDomainFileConnections(domainId);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
public getFileConnections(filePath: string) {
|
|
192
|
+
return this.dependencies.getFileConnections(filePath);
|
|
193
|
+
}
|
|
194
|
+
|
|
178
195
|
public getAllDependenciesWithDomains(): Array<{
|
|
179
196
|
source_file: string;
|
|
180
197
|
source_domain: string;
|
|
@@ -104,9 +104,10 @@ export const TABLE_STATEMENTS = [
|
|
|
104
104
|
|
|
105
105
|
export const INDEX_STATEMENTS = [
|
|
106
106
|
"CREATE INDEX IF NOT EXISTS idx_domains_name ON domains(name);",
|
|
107
|
-
"CREATE INDEX IF NOT EXISTS idx_files_domain_id ON files(domain_id);",
|
|
107
|
+
"CREATE INDEX IF NOT EXISTS idx_files_domain_id ON files(domain_id, id);",
|
|
108
108
|
"CREATE INDEX IF NOT EXISTS idx_files_path ON files(path);",
|
|
109
109
|
"CREATE INDEX IF NOT EXISTS idx_symbols_file_id ON symbols(file_id);",
|
|
110
|
+
"CREATE INDEX IF NOT EXISTS idx_symbols_file_lines ON symbols(file_id, line_start, line_end);",
|
|
110
111
|
"CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);",
|
|
111
112
|
"CREATE INDEX IF NOT EXISTS idx_symbols_kind ON symbols(kind);",
|
|
112
113
|
"CREATE INDEX IF NOT EXISTS idx_symbols_line_count ON symbols(kind, line_count DESC);",
|
|
@@ -117,7 +118,7 @@ export const INDEX_STATEMENTS = [
|
|
|
117
118
|
"CREATE INDEX IF NOT EXISTS idx_slices_route_name ON vertical_slices(route_name);",
|
|
118
119
|
"CREATE INDEX IF NOT EXISTS idx_slices_controller ON vertical_slices(controller_class, action_name);",
|
|
119
120
|
"CREATE INDEX IF NOT EXISTS idx_env_topology_status ON environment_topology(status);",
|
|
120
|
-
"CREATE INDEX IF NOT EXISTS idx_env_nodes_layer ON environment_nodes(layer);",
|
|
121
|
+
"CREATE INDEX IF NOT EXISTS idx_env_nodes_layer ON environment_nodes(layer, id);",
|
|
121
122
|
"CREATE INDEX IF NOT EXISTS idx_env_nodes_status ON environment_nodes(status);",
|
|
122
123
|
];
|
|
123
124
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
-
import { basename, dirname, join, relative, resolve } from "node:path";
|
|
2
|
+
import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
3
3
|
import type { ParsedFileAST } from "../../types/index.ts";
|
|
4
4
|
import type { ValidatedSeptumConfig } from "../config/schema.ts";
|
|
5
5
|
import type { SeptumRepository } from "../database/repository.ts";
|
|
@@ -8,6 +8,7 @@ import { SemanticSliceExtractorRegistry } from "../parser/extractors/registry.ts
|
|
|
8
8
|
import { TopologyDetector } from "../discovery/topology-detector.ts";
|
|
9
9
|
import { ASTParserEngine } from "../parser/tree-sitter.ts";
|
|
10
10
|
import { CallGraphTracer } from "../resolver/call-graph-tracer.ts";
|
|
11
|
+
import { ModuleResolver } from "../resolver/module-resolver.ts";
|
|
11
12
|
import { computeContentHash } from "./hasher.ts";
|
|
12
13
|
|
|
13
14
|
export interface IngestionMetrics {
|
|
@@ -33,13 +34,15 @@ export class IngestionPipeline {
|
|
|
33
34
|
}
|
|
34
35
|
}
|
|
35
36
|
|
|
36
|
-
public async run(config: ValidatedSeptumConfig): Promise<IngestionMetrics> {
|
|
37
|
+
public async run(config: ValidatedSeptumConfig, options: { verifyContent?: boolean } = {}): Promise<IngestionMetrics> {
|
|
38
|
+
this.ensureParserRevision();
|
|
37
39
|
const startTime = performance.now();
|
|
38
40
|
let filesScanned = 0;
|
|
39
41
|
let filesUpdated = 0;
|
|
40
42
|
let filesSkipped = 0;
|
|
41
43
|
let symbolsIndexed = 0;
|
|
42
44
|
let dependenciesIndexed = 0;
|
|
45
|
+
const unsupportedFiles: string[] = [];
|
|
43
46
|
|
|
44
47
|
let domainNames = Object.keys(config.domains);
|
|
45
48
|
const projectRoot = this.customWorkspaceRoot || process.cwd();
|
|
@@ -59,6 +62,11 @@ export class IngestionPipeline {
|
|
|
59
62
|
domainNames = Object.keys(config.domains);
|
|
60
63
|
}
|
|
61
64
|
|
|
65
|
+
const existingFilesMap = new Map<string, import("../../types/index.ts").FileRecord>();
|
|
66
|
+
for (const f of this.repo.getAllFiles()) {
|
|
67
|
+
existingFilesMap.set(f.path, f);
|
|
68
|
+
}
|
|
69
|
+
|
|
62
70
|
for (const domainName of domainNames) {
|
|
63
71
|
const domainCfg = config.domains[domainName];
|
|
64
72
|
const domainId = this.repo.upsertDomain(domainName, domainCfg);
|
|
@@ -66,17 +74,16 @@ export class IngestionPipeline {
|
|
|
66
74
|
const absDomainRoot = resolve(workspaceRoot, domainCfg.root);
|
|
67
75
|
|
|
68
76
|
if (!existsSync(absDomainRoot)) {
|
|
77
|
+
this.repo.files.deleteFilesNotInPaths(domainId, []);
|
|
69
78
|
continue;
|
|
70
79
|
}
|
|
71
80
|
|
|
72
|
-
const normCurrentRoot = domainCfg.root.replace(/\\/g, "/").replace(/^\.\/?/, "");
|
|
73
81
|
const childDomainRoots = domainNames
|
|
74
|
-
.filter(
|
|
75
|
-
.map(
|
|
76
|
-
.filter(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return r.startsWith(normCurrentRoot + "/");
|
|
82
|
+
.filter(name => name !== domainName)
|
|
83
|
+
.map(name => relative(workspaceRoot, resolve(workspaceRoot, config.domains[name].root)).replace(/\\/g, "/"))
|
|
84
|
+
.filter(child => {
|
|
85
|
+
const fromParent = relative(absDomainRoot, resolve(workspaceRoot, child)).replace(/\\/g, "/");
|
|
86
|
+
return fromParent !== "" && fromParent !== ".." && !fromParent.startsWith("../") && !isAbsolute(fromParent);
|
|
80
87
|
});
|
|
81
88
|
|
|
82
89
|
const collectedFiles = this.collectFiles(
|
|
@@ -85,7 +92,13 @@ export class IngestionPipeline {
|
|
|
85
92
|
config.settings.ignore_patterns,
|
|
86
93
|
childDomainRoots
|
|
87
94
|
);
|
|
88
|
-
const activeRelPaths
|
|
95
|
+
const activeRelPaths = new Set<string>();
|
|
96
|
+
|
|
97
|
+
const pendingMetadataUpdates: Array<{
|
|
98
|
+
id: number;
|
|
99
|
+
mtimeMs: number;
|
|
100
|
+
sizeBytes: number;
|
|
101
|
+
}> = [];
|
|
89
102
|
|
|
90
103
|
const pendingUpdates: Array<{
|
|
91
104
|
fullPath: string;
|
|
@@ -100,7 +113,7 @@ export class IngestionPipeline {
|
|
|
100
113
|
|
|
101
114
|
for (const { fullPath, relPath } of collectedFiles) {
|
|
102
115
|
filesScanned++;
|
|
103
|
-
activeRelPaths.
|
|
116
|
+
activeRelPaths.add(relPath);
|
|
104
117
|
|
|
105
118
|
let stat;
|
|
106
119
|
try {
|
|
@@ -109,12 +122,13 @@ export class IngestionPipeline {
|
|
|
109
122
|
continue;
|
|
110
123
|
}
|
|
111
124
|
|
|
112
|
-
const mtimeMs =
|
|
125
|
+
const mtimeMs = stat.mtimeMs;
|
|
113
126
|
const sizeBytes = stat.size;
|
|
114
|
-
const existingFile =
|
|
127
|
+
const existingFile = existingFilesMap.get(relPath);
|
|
115
128
|
|
|
116
129
|
// Tier 1: Fast OS metadata check (inode mtime + size + domain match)
|
|
117
130
|
if (
|
|
131
|
+
!options.verifyContent &&
|
|
118
132
|
existingFile &&
|
|
119
133
|
existingFile.domain_id === domainId &&
|
|
120
134
|
existingFile.mtime_ms === mtimeMs &&
|
|
@@ -133,17 +147,23 @@ export class IngestionPipeline {
|
|
|
133
147
|
existingFile.domain_id === domainId &&
|
|
134
148
|
existingFile.content_hash === currentHash
|
|
135
149
|
) {
|
|
136
|
-
// Content identical despite mtime bump (e.g. touch) -
|
|
137
|
-
|
|
150
|
+
// Content identical despite mtime bump (e.g. touch) - queue metadata update for batch transaction
|
|
151
|
+
pendingMetadataUpdates.push({ id: existingFile.id, mtimeMs, sizeBytes });
|
|
138
152
|
filesSkipped++;
|
|
139
153
|
continue;
|
|
140
154
|
}
|
|
141
155
|
|
|
142
|
-
filesUpdated++;
|
|
143
156
|
const lineCount = content.split("\n").length;
|
|
144
157
|
const archetype = detectArchetype(relPath, domainCfg.archetypes ?? {});
|
|
145
158
|
const ast = await this.parserEngine.parseFile(fullPath, content);
|
|
159
|
+
if (ast.status === "unsupported") {
|
|
160
|
+
unsupportedFiles.push(relPath);
|
|
161
|
+
this.repo.deleteFileByPath(relPath);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
if (ast.status === "degraded") throw new Error(`Cannot reliably index '${relPath}': parser is degraded.`);
|
|
146
165
|
|
|
166
|
+
filesUpdated++;
|
|
147
167
|
pendingUpdates.push({
|
|
148
168
|
fullPath,
|
|
149
169
|
relPath,
|
|
@@ -158,6 +178,10 @@ export class IngestionPipeline {
|
|
|
158
178
|
|
|
159
179
|
// Batch ACID Transaction per domain
|
|
160
180
|
this.repo.runInTransaction(() => {
|
|
181
|
+
for (const meta of pendingMetadataUpdates) {
|
|
182
|
+
this.repo.updateFileMetadataOnly(meta.id, meta.mtimeMs, meta.sizeBytes);
|
|
183
|
+
}
|
|
184
|
+
|
|
161
185
|
for (const item of pendingUpdates) {
|
|
162
186
|
const fileId = this.repo.upsertFile(
|
|
163
187
|
domainId,
|
|
@@ -176,8 +200,13 @@ export class IngestionPipeline {
|
|
|
176
200
|
dependenciesIndexed += item.ast.dependencies.length;
|
|
177
201
|
}
|
|
178
202
|
|
|
179
|
-
// State Reconciliation: Prune deleted files from DB
|
|
180
|
-
|
|
203
|
+
// State Reconciliation: Prune deleted files from DB using pre-loaded existingFilesMap
|
|
204
|
+
for (const [existingPath, existingRecord] of existingFilesMap.entries()) {
|
|
205
|
+
if (existingRecord.domain_id === domainId && !activeRelPaths.has(existingPath)) {
|
|
206
|
+
this.repo.deleteFileByPath(existingPath);
|
|
207
|
+
existingFilesMap.delete(existingPath);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
181
210
|
});
|
|
182
211
|
}
|
|
183
212
|
|
|
@@ -188,29 +217,29 @@ export class IngestionPipeline {
|
|
|
188
217
|
detectedTopology.framework
|
|
189
218
|
);
|
|
190
219
|
|
|
191
|
-
|
|
220
|
+
const slicesToPersist =
|
|
221
|
+
semanticSlices.length > 0
|
|
222
|
+
? semanticSlices
|
|
223
|
+
: new CallGraphTracer(this.repo, projectRoot).traceAllSlices();
|
|
192
224
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
}
|
|
197
|
-
} else {
|
|
198
|
-
// Heuristic Call-Graph Tracer fallback for Clean/Hexagonal or generic architectures
|
|
199
|
-
const callGraphTracer = new CallGraphTracer(this.repo, projectRoot);
|
|
200
|
-
const graphSlices = callGraphTracer.traceAllSlices();
|
|
201
|
-
for (const slice of graphSlices) {
|
|
225
|
+
this.repo.runInTransaction(() => {
|
|
226
|
+
this.repo.clearVerticalSlices();
|
|
227
|
+
for (const slice of slicesToPersist) {
|
|
202
228
|
this.repo.upsertVerticalSlice(slice);
|
|
203
229
|
}
|
|
204
|
-
}
|
|
230
|
+
});
|
|
205
231
|
|
|
206
232
|
const durationMs = Math.round(performance.now() - startTime);
|
|
207
233
|
|
|
208
234
|
// Update SSOT repo_meta watermark
|
|
209
|
-
this.repo.
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
235
|
+
this.repo.runInTransaction(() => {
|
|
236
|
+
this.repo.setMeta("last_sync", new Date().toISOString());
|
|
237
|
+
this.repo.setMeta("files_scanned", String(filesScanned));
|
|
238
|
+
this.repo.setMeta("files_updated", String(filesUpdated));
|
|
239
|
+
this.repo.setMeta("symbols_indexed", String(symbolsIndexed));
|
|
240
|
+
this.repo.setMeta("dependencies_indexed", String(dependenciesIndexed));
|
|
241
|
+
this.repo.setMeta("unsupported_files", JSON.stringify(unsupportedFiles));
|
|
242
|
+
});
|
|
214
243
|
|
|
215
244
|
return {
|
|
216
245
|
domains_processed: domainNames.length,
|
|
@@ -495,15 +524,27 @@ export class IngestionPipeline {
|
|
|
495
524
|
}
|
|
496
525
|
|
|
497
526
|
private isSourceFile(path: string): boolean {
|
|
498
|
-
return
|
|
527
|
+
return ASTParserEngine.isSourceFile(path);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
private ensureParserRevision(): void {
|
|
531
|
+
const revision = "2";
|
|
532
|
+
if (this.repo.getMeta("parser_revision") === revision) return;
|
|
533
|
+
this.repo.runInTransaction(() => {
|
|
534
|
+
for (const file of this.repo.getAllFiles()) {
|
|
535
|
+
this.repo.upsertFile(file.domain_id, file.path, file.archetype, "", file.line_count, -1, -1);
|
|
536
|
+
}
|
|
537
|
+
this.repo.clearVerticalSlices();
|
|
538
|
+
this.repo.setMeta("parser_revision", revision);
|
|
539
|
+
});
|
|
499
540
|
}
|
|
500
541
|
|
|
501
542
|
/**
|
|
502
543
|
* JIT single-file ingestion: quickly verifies and updates a single file's AST,
|
|
503
544
|
* symbols, and dependencies in SQLite if its mtime or content hash has changed.
|
|
504
|
-
* Runs in < 5ms for rapid delta sync without full project scans.
|
|
505
545
|
*/
|
|
506
546
|
public async ingestFile(targetFilePath: string, config: ValidatedSeptumConfig): Promise<boolean> {
|
|
547
|
+
this.ensureParserRevision();
|
|
507
548
|
const fullPath = resolve(targetFilePath);
|
|
508
549
|
if (!existsSync(fullPath)) return false;
|
|
509
550
|
|
|
@@ -515,47 +556,31 @@ export class IngestionPipeline {
|
|
|
515
556
|
return false;
|
|
516
557
|
}
|
|
517
558
|
|
|
518
|
-
const mtimeMs =
|
|
559
|
+
const mtimeMs = stat.mtimeMs;
|
|
519
560
|
const sizeBytes = stat.size;
|
|
520
561
|
|
|
521
562
|
const projectRoot = this.customWorkspaceRoot || process.cwd();
|
|
522
563
|
const relPath = relative(projectRoot, fullPath).replace(/\\/g, "/");
|
|
523
564
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
for (const [dName, dCfg] of Object.entries(config.domains)) {
|
|
528
|
-
const absRoot = resolve(projectRoot, dCfg.root).replace(/\\/g, "/");
|
|
529
|
-
const normFullPath = fullPath.replace(/\\/g, "/");
|
|
530
|
-
if (normFullPath === absRoot || normFullPath.startsWith(absRoot + "/")) {
|
|
531
|
-
targetDomainName = dName;
|
|
532
|
-
targetDomainCfg = dCfg;
|
|
533
|
-
break;
|
|
534
|
-
}
|
|
565
|
+
if (this.shouldIgnore(relPath, config.settings.ignore_patterns ?? [])) {
|
|
566
|
+
return false;
|
|
535
567
|
}
|
|
536
568
|
|
|
569
|
+
const targetDomainName = new ModuleResolver(projectRoot, this.repo).resolveSourceDomain(fullPath, config);
|
|
570
|
+
const targetDomainCfg = targetDomainName ? config.domains[targetDomainName] : null;
|
|
571
|
+
|
|
537
572
|
if (!targetDomainName || !targetDomainCfg) {
|
|
538
|
-
|
|
539
|
-
targetDomainCfg = { root: ".", allowed_dependencies: [], forbidden_dependencies: [], archetypes: {} };
|
|
573
|
+
return false;
|
|
540
574
|
}
|
|
541
575
|
|
|
542
576
|
const domainId = this.repo.upsertDomain(targetDomainName, targetDomainCfg);
|
|
543
577
|
const existingFile = this.repo.getFileByPath(relPath);
|
|
544
578
|
|
|
545
|
-
// Tier 1: Fast OS metadata check
|
|
546
|
-
if (
|
|
547
|
-
existingFile &&
|
|
548
|
-
existingFile.mtime_ms === mtimeMs &&
|
|
549
|
-
existingFile.size_bytes === sizeBytes
|
|
550
|
-
) {
|
|
551
|
-
return false; // Completely up to date
|
|
552
|
-
}
|
|
553
|
-
|
|
554
579
|
const content = readFileSync(fullPath, "utf-8");
|
|
555
580
|
const currentHash = computeContentHash(content);
|
|
556
581
|
|
|
557
582
|
// Tier 2: Hash verification
|
|
558
|
-
if (existingFile && existingFile.content_hash === currentHash) {
|
|
583
|
+
if (existingFile && existingFile.domain_id === domainId && existingFile.content_hash === currentHash) {
|
|
559
584
|
this.repo.updateFileMetadataOnly(existingFile.id, mtimeMs, sizeBytes);
|
|
560
585
|
return false;
|
|
561
586
|
}
|
|
@@ -563,6 +588,14 @@ export class IngestionPipeline {
|
|
|
563
588
|
const lineCount = content.split("\n").length;
|
|
564
589
|
const archetype = detectArchetype(relPath, targetDomainCfg.archetypes ?? {});
|
|
565
590
|
const ast = await this.parserEngine.parseFile(fullPath, content);
|
|
591
|
+
if (ast.status === "unsupported") {
|
|
592
|
+
this.repo.deleteFileByPath(relPath);
|
|
593
|
+
const unsupported = new Set<string>(JSON.parse(this.repo.getMeta("unsupported_files") || "[]"));
|
|
594
|
+
unsupported.add(relPath);
|
|
595
|
+
this.repo.setMeta("unsupported_files", JSON.stringify([...unsupported]));
|
|
596
|
+
return false;
|
|
597
|
+
}
|
|
598
|
+
if (ast.status === "degraded") throw new Error(`Cannot reliably index '${relPath}': parser is degraded.`);
|
|
566
599
|
|
|
567
600
|
this.repo.runInTransaction(() => {
|
|
568
601
|
const fileId = this.repo.upsertFile(
|
|
@@ -576,6 +609,9 @@ export class IngestionPipeline {
|
|
|
576
609
|
);
|
|
577
610
|
this.repo.replaceFileSymbols(fileId, ast.symbols);
|
|
578
611
|
this.repo.replaceFileDependencies(fileId, ast.dependencies);
|
|
612
|
+
this.repo.clearVerticalSlices();
|
|
613
|
+
const unsupported = JSON.parse(this.repo.getMeta("unsupported_files") || "[]") as string[];
|
|
614
|
+
this.repo.setMeta("unsupported_files", JSON.stringify(unsupported.filter(path => path !== relPath)));
|
|
579
615
|
});
|
|
580
616
|
|
|
581
617
|
return true;
|