septum 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +105 -102
  3. package/bin/septum.ts +4 -4
  4. package/package.json +63 -63
  5. package/src/cli/commands/check.ts +161 -161
  6. package/src/cli/commands/hook.ts +188 -188
  7. package/src/cli/commands/ingest.ts +29 -29
  8. package/src/cli/commands/init.ts +211 -211
  9. package/src/cli/commands/locate.ts +61 -61
  10. package/src/cli/commands/query.ts +64 -64
  11. package/src/cli/commands/serve.ts +5 -5
  12. package/src/cli/commands/slice.ts +48 -90
  13. package/src/cli/commands/sync.ts +31 -31
  14. package/src/cli/commands/topology.ts +119 -0
  15. package/src/cli/index.ts +210 -199
  16. package/src/cli/templates/hooks.ts +75 -75
  17. package/src/cli/templates/rules.ts +38 -38
  18. package/src/cli/templates/skill.ts +69 -69
  19. package/src/core/boundary/evaluator.ts +232 -210
  20. package/src/core/config/loader.ts +97 -97
  21. package/src/core/config/schema.ts +39 -39
  22. package/src/core/database/client.ts +44 -44
  23. package/src/core/database/repositories/dependency.repository.ts +366 -349
  24. package/src/core/database/repositories/domain.repository.ts +218 -198
  25. package/src/core/database/repositories/environment.repository.ts +207 -0
  26. package/src/core/database/repositories/file.repository.ts +116 -106
  27. package/src/core/database/repositories/meta.repository.ts +33 -33
  28. package/src/core/database/repositories/symbol.repository.ts +443 -424
  29. package/src/core/database/repository.ts +264 -255
  30. package/src/core/database/schema.ts +34 -1
  31. package/src/core/discovery/topology-detector.ts +486 -486
  32. package/src/core/ingestion/hasher.ts +5 -5
  33. package/src/core/ingestion/pipeline.ts +584 -535
  34. package/src/core/parser/behavioral-analyzer.ts +119 -0
  35. package/src/core/parser/boundary-tracker.ts +288 -128
  36. package/src/core/parser/extractors/base.ts +62 -61
  37. package/src/core/parser/extractors/frontend.ts +194 -194
  38. package/src/core/parser/extractors/go.ts +229 -176
  39. package/src/core/parser/extractors/laravel-semantic.ts +1039 -585
  40. package/src/core/parser/extractors/mcp-cli-semantic.ts +199 -199
  41. package/src/core/parser/extractors/nestjs-semantic.ts +119 -119
  42. package/src/core/parser/extractors/php.ts +153 -149
  43. package/src/core/parser/extractors/python.ts +228 -226
  44. package/src/core/parser/extractors/registry.ts +70 -70
  45. package/src/core/parser/extractors/semantic-extractor.interface.ts +21 -21
  46. package/src/core/parser/extractors/typescript.ts +181 -177
  47. package/src/core/parser/schema/providers/laravel-migration-provider.ts +215 -0
  48. package/src/core/parser/schema/providers/prisma-schema-provider.ts +89 -0
  49. package/src/core/parser/schema/schema-provider-registry.ts +43 -0
  50. package/src/core/parser/schema/schema-provider.interface.ts +33 -0
  51. package/src/core/parser/tree-sitter.ts +41 -41
  52. package/src/core/resolver/call-graph-tracer.ts +666 -668
  53. package/src/core/resolver/module-resolver.ts +408 -342
  54. package/src/core/resolver/symbol-locator.ts +543 -543
  55. package/src/core/resolver/vertical-slice-tracer.ts +301 -252
  56. package/src/core/session/session-manager.ts +56 -56
  57. package/src/core/telemetry/telemetry.ts +86 -86
  58. package/src/core/topology/environment-detector.ts +860 -0
  59. package/src/core/topology/types.ts +83 -0
  60. package/src/index.ts +21 -18
  61. package/src/mcp/schemas.ts +578 -238
  62. package/src/mcp/server.ts +875 -656
  63. package/src/mcp/tools/check-boundary.ts +171 -152
  64. package/src/mcp/tools/get-domain-catalog.ts +145 -121
  65. package/src/mcp/tools/get-environment-topology.ts +141 -0
  66. package/src/mcp/tools/get-feature-context.ts +212 -116
  67. package/src/mcp/tools/get-symbol-hotspots.ts +60 -49
  68. package/src/mcp/tools/get-symbol-impact.ts +28 -26
  69. package/src/mcp/tools/get-symbol.ts +65 -63
  70. package/src/mcp/tools/locate-symbol.ts +171 -190
  71. package/src/mcp/tools/register-domain.ts +68 -64
  72. package/src/mcp/tools/register-feature.ts +100 -0
  73. package/src/mcp/tools/trace-vertical-slice.ts +74 -103
  74. package/src/types/index.ts +335 -299
@@ -1,535 +1,584 @@
1
- import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
2
- import { basename, dirname, join, relative, resolve } from "node:path";
3
- import type { ParsedFileAST } from "../../types/index.ts";
4
- import type { ValidatedSeptumConfig } from "../config/schema.ts";
5
- import type { SeptumRepository } from "../database/repository.ts";
6
- import { detectArchetype } from "../parser/extractors/base.ts";
7
- import { SemanticSliceExtractorRegistry } from "../parser/extractors/registry.ts";
8
- import { TopologyDetector } from "../discovery/topology-detector.ts";
9
- import { ASTParserEngine } from "../parser/tree-sitter.ts";
10
- import { CallGraphTracer } from "../resolver/call-graph-tracer.ts";
11
- import { computeContentHash } from "./hasher.ts";
12
-
13
- export interface IngestionMetrics {
14
- domains_processed: number;
15
- files_scanned: number;
16
- files_updated: number;
17
- files_skipped: number;
18
- symbols_indexed: number;
19
- dependencies_indexed: number;
20
- duration_ms: number;
21
- }
22
-
23
- export class IngestionPipeline {
24
- private repo: SeptumRepository;
25
- private parserEngine: ASTParserEngine;
26
- private customWorkspaceRoot?: string;
27
-
28
- constructor(repo: SeptumRepository, workspaceRoot?: string) {
29
- this.repo = repo;
30
- this.parserEngine = new ASTParserEngine();
31
- if (workspaceRoot) {
32
- this.customWorkspaceRoot = resolve(workspaceRoot);
33
- }
34
- }
35
-
36
- public async run(config: ValidatedSeptumConfig): Promise<IngestionMetrics> {
37
- const startTime = performance.now();
38
- let filesScanned = 0;
39
- let filesUpdated = 0;
40
- let filesSkipped = 0;
41
- let symbolsIndexed = 0;
42
- let dependenciesIndexed = 0;
43
-
44
- let domainNames = Object.keys(config.domains);
45
- const projectRoot = this.customWorkspaceRoot || process.cwd();
46
-
47
- // Auto-discovery mode if no domains explicitly declared in config
48
- if (domainNames.length === 0) {
49
- const discovered = this.autoDiscoverDomains(projectRoot);
50
- for (const [dName, dCfg] of Object.entries(discovered)) {
51
- config.domains[dName] = {
52
- root: dCfg.root,
53
- description: dCfg.description,
54
- allowed_dependencies: dCfg.allowed_dependencies || [],
55
- forbidden_dependencies: dCfg.forbidden_dependencies || [],
56
- archetypes: dCfg.archetypes || {},
57
- };
58
- }
59
- domainNames = Object.keys(config.domains);
60
- }
61
-
62
- for (const domainName of domainNames) {
63
- const domainCfg = config.domains[domainName];
64
- const domainId = this.repo.upsertDomain(domainName, domainCfg);
65
-
66
- if (!existsSync(domainCfg.root)) {
67
- continue;
68
- }
69
-
70
- const workspaceRoot = this.resolveWorkspaceRoot(config, domainCfg.root);
71
- const normCurrentRoot = domainCfg.root.replace(/\\/g, "/").replace(/^\.\/?/, "");
72
- const childDomainRoots = domainNames
73
- .filter((d) => d !== domainName)
74
- .map((d) => config.domains[d].root.replace(/\\/g, "/").replace(/^\.\/?/, ""))
75
- .filter((r) => {
76
- if (!r) return false;
77
- if (!normCurrentRoot) return true;
78
- return r.startsWith(normCurrentRoot + "/");
79
- });
80
-
81
- const collectedFiles = this.collectFiles(
82
- domainCfg.root,
83
- workspaceRoot,
84
- config.settings.ignore_patterns,
85
- childDomainRoots
86
- );
87
- const activeRelPaths: string[] = [];
88
-
89
- const pendingUpdates: Array<{
90
- fullPath: string;
91
- relPath: string;
92
- contentHash: string;
93
- lineCount: number;
94
- archetype: ReturnType<typeof detectArchetype>;
95
- mtimeMs: number;
96
- sizeBytes: number;
97
- ast: ParsedFileAST;
98
- }> = [];
99
-
100
- for (const { fullPath, relPath } of collectedFiles) {
101
- filesScanned++;
102
- activeRelPaths.push(relPath);
103
-
104
- let stat;
105
- try {
106
- stat = statSync(fullPath);
107
- } catch {
108
- continue;
109
- }
110
-
111
- const mtimeMs = Math.round(stat.mtimeMs);
112
- const sizeBytes = stat.size;
113
- const existingFile = this.repo.getFileByPath(relPath);
114
-
115
- // Tier 1: Fast OS metadata check (inode mtime + size + domain match)
116
- if (
117
- existingFile &&
118
- existingFile.domain_id === domainId &&
119
- existingFile.mtime_ms === mtimeMs &&
120
- existingFile.size_bytes === sizeBytes
121
- ) {
122
- filesSkipped++;
123
- continue;
124
- }
125
-
126
- // Tier 2: Content Hash verification
127
- const content = readFileSync(fullPath, "utf-8");
128
- const currentHash = computeContentHash(content);
129
-
130
- if (
131
- existingFile &&
132
- existingFile.domain_id === domainId &&
133
- existingFile.content_hash === currentHash
134
- ) {
135
- // Content identical despite mtime bump (e.g. touch) - update metadata without re-parsing AST
136
- this.repo.updateFileMetadataOnly(existingFile.id, mtimeMs, sizeBytes);
137
- filesSkipped++;
138
- continue;
139
- }
140
-
141
- filesUpdated++;
142
- const lineCount = content.split("\n").length;
143
- const archetype = detectArchetype(relPath, domainCfg.archetypes ?? {});
144
- const ast = await this.parserEngine.parseFile(fullPath, content);
145
-
146
- pendingUpdates.push({
147
- fullPath,
148
- relPath,
149
- contentHash: currentHash,
150
- lineCount,
151
- archetype,
152
- mtimeMs,
153
- sizeBytes,
154
- ast,
155
- });
156
- }
157
-
158
- // Batch ACID Transaction per domain
159
- this.repo.runInTransaction(() => {
160
- for (const item of pendingUpdates) {
161
- const fileId = this.repo.upsertFile(
162
- domainId,
163
- item.relPath,
164
- item.archetype,
165
- item.contentHash,
166
- item.lineCount,
167
- item.mtimeMs,
168
- item.sizeBytes
169
- );
170
-
171
- this.repo.replaceFileSymbols(fileId, item.ast.symbols);
172
- this.repo.replaceFileDependencies(fileId, item.ast.dependencies);
173
-
174
- symbolsIndexed += item.ast.symbols.length;
175
- dependenciesIndexed += item.ast.dependencies.length;
176
- }
177
-
178
- // State Reconciliation: Prune deleted files from DB
179
- this.repo.deleteFilesNotInPaths(domainId, activeRelPaths);
180
- });
181
- }
182
-
183
- // Framework-Agnostic Semantic Vertical Slice Extraction (via Registry Strategy)
184
- const detectedTopology = TopologyDetector.detect(projectRoot);
185
- const semanticSlices = await SemanticSliceExtractorRegistry.extractAllSlices(
186
- projectRoot,
187
- detectedTopology.framework
188
- );
189
-
190
- this.repo.clearVerticalSlices();
191
-
192
- if (semanticSlices.length > 0) {
193
- for (const slice of semanticSlices) {
194
- this.repo.upsertVerticalSlice(slice);
195
- }
196
- } else {
197
- // Heuristic Call-Graph Tracer fallback for Clean/Hexagonal or generic architectures
198
- const callGraphTracer = new CallGraphTracer(this.repo, projectRoot);
199
- const graphSlices = callGraphTracer.traceAllSlices();
200
- for (const slice of graphSlices) {
201
- this.repo.upsertVerticalSlice(slice);
202
- }
203
- }
204
-
205
- const durationMs = Math.round(performance.now() - startTime);
206
-
207
- // Update SSOT repo_meta watermark
208
- this.repo.setMeta("last_sync", new Date().toISOString());
209
- this.repo.setMeta("files_scanned", String(filesScanned));
210
- this.repo.setMeta("files_updated", String(filesUpdated));
211
- this.repo.setMeta("symbols_indexed", String(symbolsIndexed));
212
- this.repo.setMeta("dependencies_indexed", String(dependenciesIndexed));
213
-
214
- return {
215
- domains_processed: domainNames.length,
216
- files_scanned: filesScanned,
217
- files_updated: filesUpdated,
218
- files_skipped: filesSkipped,
219
- symbols_indexed: symbolsIndexed,
220
- dependencies_indexed: dependenciesIndexed,
221
- duration_ms: durationMs,
222
- };
223
- }
224
-
225
- public autoDiscoverDomains(projectRoot: string): Record<string, import("../../types/index.ts").DomainConfig> {
226
- const domains: Record<string, import("../../types/index.ts").DomainConfig> = {};
227
-
228
- // 1. Modular / DDD check
229
- const candidateDirs = [
230
- { name: "domains", rel: "src/domains" },
231
- { name: "features", rel: "src/features" },
232
- { name: "modules", rel: "src/modules" },
233
- { name: "domain", rel: "app/Domain" },
234
- { name: "domains_app", rel: "app/Domains" },
235
- ];
236
-
237
- for (const c of candidateDirs) {
238
- const fullDir = join(projectRoot, c.rel);
239
- if (existsSync(fullDir)) {
240
- try {
241
- const subdirs = readdirSync(fullDir, { withFileTypes: true })
242
- .filter((d) => d.isDirectory())
243
- .map((d) => d.name);
244
-
245
- if (subdirs.length > 0) {
246
- for (const sub of subdirs) {
247
- domains[sub.toLowerCase()] = {
248
- root: `${c.rel}/${sub}`,
249
- description: `${sub} domain (auto-discovered)`,
250
- allowed_dependencies: [],
251
- forbidden_dependencies: [],
252
- archetypes: {
253
- service: "*Service.*",
254
- model: "*Model.*",
255
- repository: "*Repository.*",
256
- },
257
- };
258
- }
259
- return domains;
260
- }
261
- } catch {}
262
- }
263
- }
264
-
265
- // 2. Framework: Laravel
266
- const hasArtisan = existsSync(join(projectRoot, "artisan"));
267
- let isLaravel = hasArtisan;
268
- if (!isLaravel && existsSync(join(projectRoot, "composer.json"))) {
269
- try {
270
- const comp = readFileSync(join(projectRoot, "composer.json"), "utf-8");
271
- if (comp.includes("laravel/framework")) isLaravel = true;
272
- } catch {}
273
- }
274
-
275
- if (isLaravel) {
276
- domains["app"] = {
277
- root: "app",
278
- description: "Core Laravel Application (auto-discovered)",
279
- allowed_dependencies: [],
280
- forbidden_dependencies: [],
281
- archetypes: {
282
- controller: "app/Http/Controllers/**",
283
- service: "app/Services/**",
284
- model: "app/Models/**",
285
- repository: "app/Repositories/**",
286
- action: "app/Actions/**",
287
- job: "app/Jobs/**",
288
- event: "app/Events/**",
289
- listener: "app/Listeners/**",
290
- request: "app/Http/Requests/**",
291
- middleware: "app/Http/Middleware/**",
292
- },
293
- };
294
-
295
- if (existsSync(join(projectRoot, "resources/views"))) {
296
- domains["views"] = {
297
- root: "resources/views",
298
- description: "Laravel Blade Templates and UI Views (auto-discovered)",
299
- allowed_dependencies: [],
300
- forbidden_dependencies: [],
301
- archetypes: {
302
- view: "resources/views/**/*.blade.php",
303
- },
304
- };
305
- }
306
-
307
- return domains;
308
- }
309
-
310
- // 3. Framework: NestJS
311
- const hasNestCli = existsSync(join(projectRoot, "nest-cli.json"));
312
- let isNest = hasNestCli;
313
- if (!isNest && existsSync(join(projectRoot, "package.json"))) {
314
- try {
315
- const pkg = readFileSync(join(projectRoot, "package.json"), "utf-8");
316
- if (pkg.includes("@nestjs/core")) isNest = true;
317
- } catch {}
318
- }
319
-
320
- if (isNest) {
321
- domains["src"] = {
322
- root: "src",
323
- description: "NestJS Source Domain (auto-discovered)",
324
- allowed_dependencies: [],
325
- forbidden_dependencies: [],
326
- archetypes: {
327
- controller: "src/**/*.controller.ts",
328
- service: "src/**/*.service.ts",
329
- repository: "src/**/*.repository.ts",
330
- },
331
- };
332
- return domains;
333
- }
334
-
335
- // 4. Framework: Go
336
- if (existsSync(join(projectRoot, "go.mod"))) {
337
- if (existsSync(join(projectRoot, "internal"))) {
338
- domains["internal"] = {
339
- root: "internal",
340
- description: "Internal business logic packages",
341
- };
342
- }
343
- if (existsSync(join(projectRoot, "pkg"))) {
344
- domains["pkg"] = {
345
- root: "pkg",
346
- description: "Public library packages",
347
- };
348
- }
349
- if (existsSync(join(projectRoot, "cmd"))) {
350
- domains["cmd"] = {
351
- root: "cmd",
352
- description: "Application command entry points",
353
- };
354
- }
355
- if (Object.keys(domains).length > 0) return domains;
356
- }
357
-
358
- // 5. General project fallback (src or root)
359
- if (existsSync(join(projectRoot, "src"))) {
360
- domains["src"] = {
361
- root: "src",
362
- description: "Default Source Domain (auto-discovered)",
363
- };
364
- } else {
365
- domains["root"] = {
366
- root: ".",
367
- description: "Root Domain (auto-discovered)",
368
- };
369
- }
370
-
371
- return domains;
372
- }
373
-
374
- private collectFiles(
375
- dir: string,
376
- workspaceRoot: string,
377
- ignorePatterns: string[],
378
- childDomainRoots: string[] = []
379
- ): Array<{ fullPath: string; relPath: string }> {
380
- const results: Array<{ fullPath: string; relPath: string }> = [];
381
-
382
- const walk = (currentDir: string) => {
383
- let entries: string[] = [];
384
- try {
385
- entries = readdirSync(currentDir);
386
- } catch {
387
- return;
388
- }
389
-
390
- for (const entry of entries) {
391
- const fullPath = join(currentDir, entry);
392
- const relPath = relative(workspaceRoot, fullPath).replace(/\\/g, "/");
393
-
394
- if (this.shouldIgnore(relPath, ignorePatterns)) {
395
- continue;
396
- }
397
-
398
- if (childDomainRoots.some((child) => relPath === child || relPath.startsWith(child + "/"))) {
399
- continue;
400
- }
401
-
402
- try {
403
- const stat = statSync(fullPath);
404
- if (stat.isDirectory()) {
405
- walk(fullPath);
406
- } else if (stat.isFile() && this.isSourceFile(fullPath)) {
407
- results.push({ fullPath, relPath });
408
- }
409
- } catch {
410
- // ignore unreadable files
411
- }
412
- }
413
- };
414
-
415
- walk(dir);
416
- return results;
417
- }
418
-
419
- private resolveWorkspaceRoot(config: ValidatedSeptumConfig, domainRoot: string): string {
420
- if (this.customWorkspaceRoot) {
421
- return this.customWorkspaceRoot;
422
- }
423
- const settings = config.settings as Record<string, unknown> | undefined;
424
- if (typeof settings?.workspace_root === "string" && settings.workspace_root.trim()) {
425
- return resolve(settings.workspace_root);
426
- }
427
- if (config.settings.db_path && config.settings.db_path !== ":memory:") {
428
- const dbDir = dirname(resolve(config.settings.db_path));
429
- const candidate = basename(dbDir) === ".septum" ? dirname(dbDir) : dbDir;
430
- const absDomainRoot = resolve(domainRoot);
431
- if (absDomainRoot.startsWith(candidate)) {
432
- return candidate;
433
- }
434
- }
435
- return process.cwd();
436
- }
437
-
438
- private shouldIgnore(path: string, ignorePatterns: string[]): boolean {
439
- for (const pattern of ignorePatterns) {
440
- const cleanPattern = pattern.replace(/\/\*\*|\*|\*\*/g, "");
441
- if (path.includes(cleanPattern)) {
442
- return true;
443
- }
444
- }
445
- return false;
446
- }
447
-
448
- private isSourceFile(path: string): boolean {
449
- return /\.(ts|tsx|js|jsx|mjs|cjs|php|py|go|rs|vue|astro|svelte)$/i.test(path);
450
- }
451
-
452
- /**
453
- * JIT single-file ingestion: quickly verifies and updates a single file's AST,
454
- * symbols, and dependencies in SQLite if its mtime or content hash has changed.
455
- * Runs in < 5ms for rapid delta sync without full project scans.
456
- */
457
- public async ingestFile(targetFilePath: string, config: ValidatedSeptumConfig): Promise<boolean> {
458
- const fullPath = resolve(targetFilePath);
459
- if (!existsSync(fullPath)) return false;
460
-
461
- let stat;
462
- try {
463
- stat = statSync(fullPath);
464
- if (!stat.isFile() || !this.isSourceFile(fullPath)) return false;
465
- } catch {
466
- return false;
467
- }
468
-
469
- const mtimeMs = Math.round(stat.mtimeMs);
470
- const sizeBytes = stat.size;
471
-
472
- const projectRoot = this.customWorkspaceRoot || process.cwd();
473
- const relPath = relative(projectRoot, fullPath).replace(/\\/g, "/");
474
-
475
- let targetDomainName: string | null = null;
476
- let targetDomainCfg = null;
477
-
478
- for (const [dName, dCfg] of Object.entries(config.domains)) {
479
- const absRoot = resolve(projectRoot, dCfg.root).replace(/\\/g, "/");
480
- const normFullPath = fullPath.replace(/\\/g, "/");
481
- if (normFullPath === absRoot || normFullPath.startsWith(absRoot + "/")) {
482
- targetDomainName = dName;
483
- targetDomainCfg = dCfg;
484
- break;
485
- }
486
- }
487
-
488
- if (!targetDomainName || !targetDomainCfg) {
489
- targetDomainName = "core";
490
- targetDomainCfg = { root: ".", allowed_dependencies: [], forbidden_dependencies: [], archetypes: {} };
491
- }
492
-
493
- const domainId = this.repo.upsertDomain(targetDomainName, targetDomainCfg);
494
- const existingFile = this.repo.getFileByPath(relPath);
495
-
496
- // Tier 1: Fast OS metadata check
497
- if (
498
- existingFile &&
499
- existingFile.mtime_ms === mtimeMs &&
500
- existingFile.size_bytes === sizeBytes
501
- ) {
502
- return false; // Completely up to date
503
- }
504
-
505
- const content = readFileSync(fullPath, "utf-8");
506
- const currentHash = computeContentHash(content);
507
-
508
- // Tier 2: Hash verification
509
- if (existingFile && existingFile.content_hash === currentHash) {
510
- this.repo.updateFileMetadataOnly(existingFile.id, mtimeMs, sizeBytes);
511
- return false;
512
- }
513
-
514
- const lineCount = content.split("\n").length;
515
- const archetype = detectArchetype(relPath, targetDomainCfg.archetypes ?? {});
516
- const ast = await this.parserEngine.parseFile(fullPath, content);
517
-
518
- this.repo.runInTransaction(() => {
519
- const fileId = this.repo.upsertFile(
520
- domainId,
521
- relPath,
522
- archetype,
523
- currentHash,
524
- lineCount,
525
- mtimeMs,
526
- sizeBytes
527
- );
528
- this.repo.replaceFileSymbols(fileId, ast.symbols);
529
- this.repo.replaceFileDependencies(fileId, ast.dependencies);
530
- });
531
-
532
- return true;
533
- }
534
- }
535
-
1
+ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
2
+ import { basename, dirname, join, relative, resolve } from "node:path";
3
+ import type { ParsedFileAST } from "../../types/index.ts";
4
+ import type { ValidatedSeptumConfig } from "../config/schema.ts";
5
+ import type { SeptumRepository } from "../database/repository.ts";
6
+ import { detectArchetype } from "../parser/extractors/base.ts";
7
+ import { SemanticSliceExtractorRegistry } from "../parser/extractors/registry.ts";
8
+ import { TopologyDetector } from "../discovery/topology-detector.ts";
9
+ import { ASTParserEngine } from "../parser/tree-sitter.ts";
10
+ import { CallGraphTracer } from "../resolver/call-graph-tracer.ts";
11
+ import { computeContentHash } from "./hasher.ts";
12
+
13
+ export interface IngestionMetrics {
14
+ domains_processed: number;
15
+ files_scanned: number;
16
+ files_updated: number;
17
+ files_skipped: number;
18
+ symbols_indexed: number;
19
+ dependencies_indexed: number;
20
+ duration_ms: number;
21
+ }
22
+
23
+ export class IngestionPipeline {
24
+ private repo: SeptumRepository;
25
+ private parserEngine: ASTParserEngine;
26
+ private customWorkspaceRoot?: string;
27
+
28
+ constructor(repo: SeptumRepository, workspaceRoot?: string) {
29
+ this.repo = repo;
30
+ this.parserEngine = new ASTParserEngine();
31
+ if (workspaceRoot) {
32
+ this.customWorkspaceRoot = resolve(workspaceRoot);
33
+ }
34
+ }
35
+
36
+ public async run(config: ValidatedSeptumConfig): Promise<IngestionMetrics> {
37
+ const startTime = performance.now();
38
+ let filesScanned = 0;
39
+ let filesUpdated = 0;
40
+ let filesSkipped = 0;
41
+ let symbolsIndexed = 0;
42
+ let dependenciesIndexed = 0;
43
+
44
+ let domainNames = Object.keys(config.domains);
45
+ const projectRoot = this.customWorkspaceRoot || process.cwd();
46
+
47
+ // Auto-discovery mode if no domains explicitly declared in config
48
+ if (domainNames.length === 0) {
49
+ const discovered = this.autoDiscoverDomains(projectRoot);
50
+ for (const [dName, dCfg] of Object.entries(discovered)) {
51
+ config.domains[dName] = {
52
+ root: dCfg.root,
53
+ description: dCfg.description,
54
+ allowed_dependencies: dCfg.allowed_dependencies || [],
55
+ forbidden_dependencies: dCfg.forbidden_dependencies || [],
56
+ archetypes: dCfg.archetypes || {},
57
+ };
58
+ }
59
+ domainNames = Object.keys(config.domains);
60
+ }
61
+
62
+ for (const domainName of domainNames) {
63
+ const domainCfg = config.domains[domainName];
64
+ const domainId = this.repo.upsertDomain(domainName, domainCfg);
65
+ const workspaceRoot = this.resolveWorkspaceRoot(config, domainCfg.root);
66
+ const absDomainRoot = resolve(workspaceRoot, domainCfg.root);
67
+
68
+ if (!existsSync(absDomainRoot)) {
69
+ continue;
70
+ }
71
+
72
+ const normCurrentRoot = domainCfg.root.replace(/\\/g, "/").replace(/^\.\/?/, "");
73
+ const childDomainRoots = domainNames
74
+ .filter((d) => d !== domainName)
75
+ .map((d) => config.domains[d].root.replace(/\\/g, "/").replace(/^\.\/?/, ""))
76
+ .filter((r) => {
77
+ if (!r) return false;
78
+ if (!normCurrentRoot) return true;
79
+ return r.startsWith(normCurrentRoot + "/");
80
+ });
81
+
82
+ const collectedFiles = this.collectFiles(
83
+ absDomainRoot,
84
+ workspaceRoot,
85
+ config.settings.ignore_patterns,
86
+ childDomainRoots
87
+ );
88
+ const activeRelPaths: string[] = [];
89
+
90
+ const pendingUpdates: Array<{
91
+ fullPath: string;
92
+ relPath: string;
93
+ contentHash: string;
94
+ lineCount: number;
95
+ archetype: ReturnType<typeof detectArchetype>;
96
+ mtimeMs: number;
97
+ sizeBytes: number;
98
+ ast: ParsedFileAST;
99
+ }> = [];
100
+
101
+ for (const { fullPath, relPath } of collectedFiles) {
102
+ filesScanned++;
103
+ activeRelPaths.push(relPath);
104
+
105
+ let stat;
106
+ try {
107
+ stat = statSync(fullPath);
108
+ } catch {
109
+ continue;
110
+ }
111
+
112
+ const mtimeMs = Math.round(stat.mtimeMs);
113
+ const sizeBytes = stat.size;
114
+ const existingFile = this.repo.getFileByPath(relPath);
115
+
116
+ // Tier 1: Fast OS metadata check (inode mtime + size + domain match)
117
+ if (
118
+ existingFile &&
119
+ existingFile.domain_id === domainId &&
120
+ existingFile.mtime_ms === mtimeMs &&
121
+ existingFile.size_bytes === sizeBytes
122
+ ) {
123
+ filesSkipped++;
124
+ continue;
125
+ }
126
+
127
+ // Tier 2: Content Hash verification
128
+ const content = readFileSync(fullPath, "utf-8");
129
+ const currentHash = computeContentHash(content);
130
+
131
+ if (
132
+ existingFile &&
133
+ existingFile.domain_id === domainId &&
134
+ existingFile.content_hash === currentHash
135
+ ) {
136
+ // Content identical despite mtime bump (e.g. touch) - update metadata without re-parsing AST
137
+ this.repo.updateFileMetadataOnly(existingFile.id, mtimeMs, sizeBytes);
138
+ filesSkipped++;
139
+ continue;
140
+ }
141
+
142
+ filesUpdated++;
143
+ const lineCount = content.split("\n").length;
144
+ const archetype = detectArchetype(relPath, domainCfg.archetypes ?? {});
145
+ const ast = await this.parserEngine.parseFile(fullPath, content);
146
+
147
+ pendingUpdates.push({
148
+ fullPath,
149
+ relPath,
150
+ contentHash: currentHash,
151
+ lineCount,
152
+ archetype,
153
+ mtimeMs,
154
+ sizeBytes,
155
+ ast,
156
+ });
157
+ }
158
+
159
+ // Batch ACID Transaction per domain
160
+ this.repo.runInTransaction(() => {
161
+ for (const item of pendingUpdates) {
162
+ const fileId = this.repo.upsertFile(
163
+ domainId,
164
+ item.relPath,
165
+ item.archetype,
166
+ item.contentHash,
167
+ item.lineCount,
168
+ item.mtimeMs,
169
+ item.sizeBytes
170
+ );
171
+
172
+ this.repo.replaceFileSymbols(fileId, item.ast.symbols);
173
+ this.repo.replaceFileDependencies(fileId, item.ast.dependencies);
174
+
175
+ symbolsIndexed += item.ast.symbols.length;
176
+ dependenciesIndexed += item.ast.dependencies.length;
177
+ }
178
+
179
+ // State Reconciliation: Prune deleted files from DB
180
+ this.repo.deleteFilesNotInPaths(domainId, activeRelPaths);
181
+ });
182
+ }
183
+
184
+ // Framework-Agnostic Semantic Vertical Slice Extraction (via Registry Strategy)
185
+ const detectedTopology = TopologyDetector.detect(projectRoot);
186
+ const semanticSlices = await SemanticSliceExtractorRegistry.extractAllSlices(
187
+ projectRoot,
188
+ detectedTopology.framework
189
+ );
190
+
191
+ this.repo.clearVerticalSlices();
192
+
193
+ if (semanticSlices.length > 0) {
194
+ for (const slice of semanticSlices) {
195
+ this.repo.upsertVerticalSlice(slice);
196
+ }
197
+ } else {
198
+ // Heuristic Call-Graph Tracer fallback for Clean/Hexagonal or generic architectures
199
+ const callGraphTracer = new CallGraphTracer(this.repo, projectRoot);
200
+ const graphSlices = callGraphTracer.traceAllSlices();
201
+ for (const slice of graphSlices) {
202
+ this.repo.upsertVerticalSlice(slice);
203
+ }
204
+ }
205
+
206
+ const durationMs = Math.round(performance.now() - startTime);
207
+
208
+ // Update SSOT repo_meta watermark
209
+ this.repo.setMeta("last_sync", new Date().toISOString());
210
+ this.repo.setMeta("files_scanned", String(filesScanned));
211
+ this.repo.setMeta("files_updated", String(filesUpdated));
212
+ this.repo.setMeta("symbols_indexed", String(symbolsIndexed));
213
+ this.repo.setMeta("dependencies_indexed", String(dependenciesIndexed));
214
+
215
+ return {
216
+ domains_processed: domainNames.length,
217
+ files_scanned: filesScanned,
218
+ files_updated: filesUpdated,
219
+ files_skipped: filesSkipped,
220
+ symbols_indexed: symbolsIndexed,
221
+ dependencies_indexed: dependenciesIndexed,
222
+ duration_ms: durationMs,
223
+ };
224
+ }
225
+
226
+ public autoDiscoverDomains(projectRoot: string): Record<string, import("../../types/index.ts").DomainConfig> {
227
+ const domains: Record<string, import("../../types/index.ts").DomainConfig> = {};
228
+
229
+ // 1. Modular / DDD check
230
+ const candidateDirs = [
231
+ { name: "domains", rel: "src/domains" },
232
+ { name: "features", rel: "src/features" },
233
+ { name: "modules", rel: "src/modules" },
234
+ { name: "domain", rel: "app/Domain" },
235
+ { name: "domains_app", rel: "app/Domains" },
236
+ ];
237
+
238
+ for (const c of candidateDirs) {
239
+ const fullDir = join(projectRoot, c.rel);
240
+ if (existsSync(fullDir)) {
241
+ try {
242
+ const subdirs = readdirSync(fullDir, { withFileTypes: true })
243
+ .filter((d) => d.isDirectory())
244
+ .map((d) => d.name);
245
+
246
+ if (subdirs.length > 0) {
247
+ for (const sub of subdirs) {
248
+ domains[sub.toLowerCase()] = {
249
+ root: `${c.rel}/${sub}`,
250
+ description: `${sub} domain (auto-discovered)`,
251
+ allowed_dependencies: [],
252
+ forbidden_dependencies: [],
253
+ archetypes: {
254
+ service: "*Service.*",
255
+ model: "*Model.*",
256
+ repository: "*Repository.*",
257
+ },
258
+ };
259
+ }
260
+ return domains;
261
+ }
262
+ } catch {}
263
+ }
264
+ }
265
+
266
+ // 2. Framework: Laravel
267
+ const hasArtisan = existsSync(join(projectRoot, "artisan"));
268
+ let isLaravel = hasArtisan;
269
+ if (!isLaravel && existsSync(join(projectRoot, "composer.json"))) {
270
+ try {
271
+ const comp = readFileSync(join(projectRoot, "composer.json"), "utf-8");
272
+ if (comp.includes("laravel/framework")) isLaravel = true;
273
+ } catch {}
274
+ }
275
+
276
+ if (isLaravel) {
277
+ domains["app"] = {
278
+ root: "app",
279
+ description: "Core Laravel Application (auto-discovered)",
280
+ allowed_dependencies: [],
281
+ forbidden_dependencies: [],
282
+ archetypes: {
283
+ controller: "app/Http/Controllers/**",
284
+ service: "app/Services/**",
285
+ model: "app/Models/**",
286
+ repository: "app/Repositories/**",
287
+ action: "app/Actions/**",
288
+ job: "app/Jobs/**",
289
+ event: "app/Events/**",
290
+ listener: "app/Listeners/**",
291
+ request: "app/Http/Requests/**",
292
+ middleware: "app/Http/Middleware/**",
293
+ },
294
+ };
295
+
296
+ if (existsSync(join(projectRoot, "resources/views"))) {
297
+ domains["views"] = {
298
+ root: "resources/views",
299
+ description: "Laravel Blade Templates and UI Views (auto-discovered)",
300
+ allowed_dependencies: [],
301
+ forbidden_dependencies: [],
302
+ archetypes: {
303
+ view: "resources/views/**/*.blade.php",
304
+ },
305
+ };
306
+ }
307
+
308
+ if (existsSync(join(projectRoot, "routes"))) {
309
+ domains["routes"] = {
310
+ root: "routes",
311
+ description: "Laravel HTTP Route Definitions (auto-discovered)",
312
+ allowed_dependencies: [],
313
+ forbidden_dependencies: [],
314
+ archetypes: {
315
+ route: "routes/**/*.php",
316
+ },
317
+ };
318
+ }
319
+
320
+ if (existsSync(join(projectRoot, "database"))) {
321
+ domains["database"] = {
322
+ root: "database",
323
+ description: "Laravel Migrations and Database Schemas (auto-discovered)",
324
+ allowed_dependencies: [],
325
+ forbidden_dependencies: [],
326
+ archetypes: {
327
+ migration: "database/migrations/**/*.php",
328
+ },
329
+ };
330
+ }
331
+
332
+ return domains;
333
+ }
334
+
335
+ // 3. Framework: NestJS
336
+ const hasNestCli = existsSync(join(projectRoot, "nest-cli.json"));
337
+ let isNest = hasNestCli;
338
+ if (!isNest && existsSync(join(projectRoot, "package.json"))) {
339
+ try {
340
+ const pkg = readFileSync(join(projectRoot, "package.json"), "utf-8");
341
+ if (pkg.includes("@nestjs/core")) isNest = true;
342
+ } catch {}
343
+ }
344
+
345
+ if (isNest) {
346
+ domains["src"] = {
347
+ root: "src",
348
+ description: "NestJS Source Domain (auto-discovered)",
349
+ allowed_dependencies: [],
350
+ forbidden_dependencies: [],
351
+ archetypes: {
352
+ controller: "src/**/*.controller.ts",
353
+ service: "src/**/*.service.ts",
354
+ repository: "src/**/*.repository.ts",
355
+ },
356
+ };
357
+ return domains;
358
+ }
359
+
360
+ // 4. Framework: Go
361
+ if (existsSync(join(projectRoot, "go.mod"))) {
362
+ if (existsSync(join(projectRoot, "internal"))) {
363
+ domains["internal"] = {
364
+ root: "internal",
365
+ description: "Internal business logic packages",
366
+ };
367
+ }
368
+ if (existsSync(join(projectRoot, "pkg"))) {
369
+ domains["pkg"] = {
370
+ root: "pkg",
371
+ description: "Public library packages",
372
+ };
373
+ }
374
+ if (existsSync(join(projectRoot, "cmd"))) {
375
+ domains["cmd"] = {
376
+ root: "cmd",
377
+ description: "Application command entry points",
378
+ };
379
+ }
380
+ if (Object.keys(domains).length > 0) return domains;
381
+ }
382
+
383
+ // 5. General project fallback (src or root)
384
+ if (existsSync(join(projectRoot, "src"))) {
385
+ domains["src"] = {
386
+ root: "src",
387
+ description: "Default Source Domain (auto-discovered)",
388
+ };
389
+ } else {
390
+ domains["root"] = {
391
+ root: ".",
392
+ description: "Root Domain (auto-discovered)",
393
+ };
394
+ }
395
+
396
+ return domains;
397
+ }
398
+
399
+ private collectFiles(
400
+ dir: string,
401
+ workspaceRoot: string,
402
+ ignorePatterns: string[],
403
+ childDomainRoots: string[] = []
404
+ ): Array<{ fullPath: string; relPath: string }> {
405
+ const results: Array<{ fullPath: string; relPath: string }> = [];
406
+
407
+ const walk = (currentDir: string) => {
408
+ let entries: string[] = [];
409
+ try {
410
+ entries = readdirSync(currentDir);
411
+ } catch {
412
+ return;
413
+ }
414
+
415
+ for (const entry of entries) {
416
+ const fullPath = join(currentDir, entry);
417
+ const relPath = relative(workspaceRoot, fullPath).replace(/\\/g, "/");
418
+
419
+ if (this.shouldIgnore(relPath, ignorePatterns)) {
420
+ continue;
421
+ }
422
+
423
+ if (childDomainRoots.some((child) => relPath === child || relPath.startsWith(child + "/"))) {
424
+ continue;
425
+ }
426
+
427
+ try {
428
+ const stat = statSync(fullPath);
429
+ if (stat.isDirectory()) {
430
+ walk(fullPath);
431
+ } else if (stat.isFile() && this.isSourceFile(fullPath)) {
432
+ results.push({ fullPath, relPath });
433
+ }
434
+ } catch {
435
+ // ignore unreadable files
436
+ }
437
+ }
438
+ };
439
+
440
+ walk(dir);
441
+ return results;
442
+ }
443
+
444
+ private resolveWorkspaceRoot(config: ValidatedSeptumConfig, domainRoot: string): string {
445
+ if (this.customWorkspaceRoot) {
446
+ return this.customWorkspaceRoot;
447
+ }
448
+ const settings = config.settings as Record<string, unknown> | undefined;
449
+ if (typeof settings?.workspace_root === "string" && settings.workspace_root.trim()) {
450
+ return resolve(settings.workspace_root);
451
+ }
452
+ if (config.settings.db_path && config.settings.db_path !== ":memory:") {
453
+ const dbDir = dirname(resolve(config.settings.db_path));
454
+ const candidate = basename(dbDir) === ".septum" ? dirname(dbDir) : dbDir;
455
+ const absDomainRoot = resolve(domainRoot);
456
+ if (absDomainRoot.startsWith(candidate)) {
457
+ return candidate;
458
+ }
459
+ }
460
+ return process.cwd();
461
+ }
462
+
463
+ private shouldIgnore(filePath: string, ignorePatterns: string[]): boolean {
464
+ const normPath = filePath.replace(/\\/g, "/").replace(/^\.\//, "");
465
+ for (const pattern of ignorePatterns) {
466
+ const normPattern = pattern.replace(/\\/g, "/").replace(/^\.\//, "").trim();
467
+ if (!normPattern) continue;
468
+
469
+ if (normPattern === "*" || normPattern === "**") {
470
+ return true;
471
+ }
472
+
473
+ if (normPath === normPattern) return true;
474
+
475
+ if (normPattern.endsWith("/**")) {
476
+ const prefix = normPattern.slice(0, -3);
477
+ if (normPath === prefix || normPath.startsWith(prefix + "/")) return true;
478
+ } else if (normPattern.endsWith("/*")) {
479
+ const prefix = normPattern.slice(0, -2);
480
+ if (normPath === prefix || normPath.startsWith(prefix + "/")) return true;
481
+ } else if (normPath.startsWith(normPattern + "/")) {
482
+ return true;
483
+ }
484
+
485
+ if (normPattern.startsWith("*.") && normPath.endsWith(normPattern.slice(1))) {
486
+ return true;
487
+ }
488
+
489
+ const segments = normPath.split("/");
490
+ if (segments.includes(normPattern)) {
491
+ return true;
492
+ }
493
+ }
494
+ return false;
495
+ }
496
+
497
+ private isSourceFile(path: string): boolean {
498
+ return /\.(ts|tsx|js|jsx|mjs|cjs|php|py|go|rs|vue|astro|svelte)$/i.test(path);
499
+ }
500
+
501
+ /**
502
+ * JIT single-file ingestion: quickly verifies and updates a single file's AST,
503
+ * symbols, and dependencies in SQLite if its mtime or content hash has changed.
504
+ * Runs in < 5ms for rapid delta sync without full project scans.
505
+ */
506
+ public async ingestFile(targetFilePath: string, config: ValidatedSeptumConfig): Promise<boolean> {
507
+ const fullPath = resolve(targetFilePath);
508
+ if (!existsSync(fullPath)) return false;
509
+
510
+ let stat;
511
+ try {
512
+ stat = statSync(fullPath);
513
+ if (!stat.isFile() || !this.isSourceFile(fullPath)) return false;
514
+ } catch {
515
+ return false;
516
+ }
517
+
518
+ const mtimeMs = Math.round(stat.mtimeMs);
519
+ const sizeBytes = stat.size;
520
+
521
+ const projectRoot = this.customWorkspaceRoot || process.cwd();
522
+ const relPath = relative(projectRoot, fullPath).replace(/\\/g, "/");
523
+
524
+ let targetDomainName: string | null = null;
525
+ let targetDomainCfg = null;
526
+
527
+ for (const [dName, dCfg] of Object.entries(config.domains)) {
528
+ const absRoot = resolve(projectRoot, dCfg.root).replace(/\\/g, "/");
529
+ const normFullPath = fullPath.replace(/\\/g, "/");
530
+ if (normFullPath === absRoot || normFullPath.startsWith(absRoot + "/")) {
531
+ targetDomainName = dName;
532
+ targetDomainCfg = dCfg;
533
+ break;
534
+ }
535
+ }
536
+
537
+ if (!targetDomainName || !targetDomainCfg) {
538
+ targetDomainName = "core";
539
+ targetDomainCfg = { root: ".", allowed_dependencies: [], forbidden_dependencies: [], archetypes: {} };
540
+ }
541
+
542
+ const domainId = this.repo.upsertDomain(targetDomainName, targetDomainCfg);
543
+ const existingFile = this.repo.getFileByPath(relPath);
544
+
545
+ // Tier 1: Fast OS metadata check
546
+ if (
547
+ existingFile &&
548
+ existingFile.mtime_ms === mtimeMs &&
549
+ existingFile.size_bytes === sizeBytes
550
+ ) {
551
+ return false; // Completely up to date
552
+ }
553
+
554
+ const content = readFileSync(fullPath, "utf-8");
555
+ const currentHash = computeContentHash(content);
556
+
557
+ // Tier 2: Hash verification
558
+ if (existingFile && existingFile.content_hash === currentHash) {
559
+ this.repo.updateFileMetadataOnly(existingFile.id, mtimeMs, sizeBytes);
560
+ return false;
561
+ }
562
+
563
+ const lineCount = content.split("\n").length;
564
+ const archetype = detectArchetype(relPath, targetDomainCfg.archetypes ?? {});
565
+ const ast = await this.parserEngine.parseFile(fullPath, content);
566
+
567
+ this.repo.runInTransaction(() => {
568
+ const fileId = this.repo.upsertFile(
569
+ domainId,
570
+ relPath,
571
+ archetype,
572
+ currentHash,
573
+ lineCount,
574
+ mtimeMs,
575
+ sizeBytes
576
+ );
577
+ this.repo.replaceFileSymbols(fileId, ast.symbols);
578
+ this.repo.replaceFileDependencies(fileId, ast.dependencies);
579
+ });
580
+
581
+ return true;
582
+ }
583
+ }
584
+