septum 0.1.6 → 0.1.8

package/src/core/ingestion/pipeline.ts

@@ -1,5 +1,9 @@
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { basename, dirname, join, relative, resolve } from "node:path";
+import { shouldIgnoreSourcePath } from "../resources/ignore.ts";
+import { RESOURCE_LIMITS } from "../resources/limits.ts";
+import { readSourceFile, SourceFileLimitError, collectSourceSkips, type SkippedSourceFile, type SourceReadFingerprint } from "../resources/source-files.ts";
+import { existsSync, readdirSync, readFileSync, statSync, lstatSync, realpathSync } from "node:fs";
+import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
+import { isPathContained, resolveWorkspaceRoot } from "../resolver/path-resolver.ts";
 import type { ParsedFileAST } from "../../types/index.ts";
 import type { ValidatedSeptumConfig } from "../config/schema.ts";
 import type { SeptumRepository } from "../database/repository.ts";
@@ -8,6 +12,7 @@ import { SemanticSliceExtractorRegistry } from "../parser/extractors/registry.ts
 import { TopologyDetector } from "../discovery/topology-detector.ts";
 import { ASTParserEngine } from "../parser/tree-sitter.ts";
 import { CallGraphTracer } from "../resolver/call-graph-tracer.ts";
+import { ModuleResolver } from "../resolver/module-resolver.ts";
 import { computeContentHash } from "./hasher.ts";
 
 export interface IngestionMetrics {
@@ -33,16 +38,34 @@ export class IngestionPipeline {
     }
   }
 
-  public async run(config: ValidatedSeptumConfig): Promise<IngestionMetrics> {
+  public async run(config: ValidatedSeptumConfig, options: { verifyContent?: boolean; signal?: AbortSignal } = {}): Promise<IngestionMetrics> {
+    this.repo.setMeta("sync_state", "running");
+    try {
+      const result = await this.runSync(config, options);
+      this.repo.setMeta("sync_state", "complete");
+      this.repo.setMeta("sync_error", "");
+      return result;
+    } catch (error) {
+      this.repo.setMeta("sync_state", "error");
+      this.repo.setMeta("sync_error", String(error));
+      throw error;
+    }
+  }
+
+  private async runSync(config: ValidatedSeptumConfig, options: { verifyContent?: boolean; signal?: AbortSignal }): Promise<IngestionMetrics> {
+    this.ensureParserRevision();
     const startTime = performance.now();
     let filesScanned = 0;
     let filesUpdated = 0;
     let filesSkipped = 0;
     let symbolsIndexed = 0;
     let dependenciesIndexed = 0;
+    const unsupportedFiles: string[] = [];
+    const skippedFiles: SkippedSourceFile[] = [];
+    let removedFiles = 0;
 
     let domainNames = Object.keys(config.domains);
-    const projectRoot = this.customWorkspaceRoot || process.cwd();
+    const projectRoot = this.customWorkspaceRoot || resolveWorkspaceRoot();
 
     // Auto-discovery mode if no domains explicitly declared in config
     if (domainNames.length === 0) {
@@ -68,27 +91,42 @@ export class IngestionPipeline {
       const domainCfg = config.domains[domainName];
       const domainId = this.repo.upsertDomain(domainName, domainCfg);
       const workspaceRoot = this.resolveWorkspaceRoot(config, domainCfg.root);
+      // SV-02: Enforce workspace containment during ingestion (reject traversal/symlink escape)
+      if (!isPathContained(domainCfg.root, workspaceRoot)) {
+        throw new Error(`Domain root '${domainCfg.root}' for domain '${domainName}' escapes workspace boundary '${workspaceRoot}'.`);
+      }
       const absDomainRoot = resolve(workspaceRoot, domainCfg.root);
 
-      if (!existsSync(absDomainRoot)) {
-        continue;
+      try {
+        const rootStat = statSync(absDomainRoot);
+        if (!rootStat.isDirectory()) {
+          throw new Error(`Domain root '${domainCfg.root}' is not a directory.`);
+        }
+      } catch (err: any) {
+        if (err?.code === "ENOENT") {
+          removedFiles += this.repo.getAllFiles().filter(file => file.domain_id === domainId).length;
+          this.repo.files.deleteFilesNotInPaths(domainId, []);
+          if (removedFiles) this.repo.setMeta("slices_dirty", "1");
+          continue;
+        }
+        // SV-01: Permission or I/O failure (EACCES, EPERM, EIO). Do NOT prune, propagate error!
+        throw new Error(`Failed to access domain root '${domainCfg.root}' for domain '${domainName}': ${err.message || err}`);
       }
 
-      const normCurrentRoot = domainCfg.root.replace(/\\/g, "/").replace(/^\.\/?/, "");
       const childDomainRoots = domainNames
-        .filter((d) => d !== domainName)
-        .map((d) => config.domains[d].root.replace(/\\/g, "/").replace(/^\.\/?/, ""))
-        .filter((r) => {
-          if (!r) return false;
-          if (!normCurrentRoot) return true;
-          return r.startsWith(normCurrentRoot + "/");
+        .filter(name => name !== domainName)
+        .map(name => relative(workspaceRoot, resolve(workspaceRoot, config.domains[name].root)).replace(/\\/g, "/"))
+        .filter(child => {
+          const fromParent = relative(absDomainRoot, resolve(workspaceRoot, child)).replace(/\\/g, "/");
+          return fromParent !== "" && fromParent !== ".." && !fromParent.startsWith("../") && !isAbsolute(fromParent);
         });
 
       const collectedFiles = this.collectFiles(
         absDomainRoot,
         workspaceRoot,
         config.settings.ignore_patterns,
-        childDomainRoots
+        childDomainRoots,
+        options.signal
       );
       const activeRelPaths = new Set<string>();
 
@@ -109,8 +147,43 @@ export class IngestionPipeline {
         ast: ParsedFileAST;
       }> = [];
 
+      let batchBytes = 0;
+      const commitBatch = () => {
+        this.repo.runInTransaction(() => {
+          for (const meta of pendingMetadataUpdates) {
+            this.repo.updateFileMetadataOnly(meta.id, meta.mtimeMs, meta.sizeBytes);
+          }
+
+          if (pendingUpdates.length) this.repo.setMeta("slices_dirty", "1");
+          for (const item of pendingUpdates) {
+            const fileId = this.repo.upsertFile(
+              domainId,
+              item.relPath,
+              item.archetype,
+              item.contentHash,
+              item.lineCount,
+              item.mtimeMs,
+              item.sizeBytes
+            );
+
+            this.repo.replaceFileSymbols(fileId, item.ast.symbols);
+            this.repo.replaceFileDependencies(fileId, item.ast.dependencies);
+
+            symbolsIndexed += item.ast.symbols.length;
+            dependenciesIndexed += item.ast.dependencies.length;
+          }
+
+        });
+        pendingUpdates.length = 0;
+        pendingMetadataUpdates.length = 0;
+        batchBytes = 0;
+      };
+
       for (const { fullPath, relPath } of collectedFiles) {
+        options.signal?.throwIfAborted();
+        if (pendingMetadataUpdates.length >= RESOURCE_LIMITS.batchFiles) commitBatch();
         filesScanned++;
+        if (filesScanned % RESOURCE_LIMITS.batchFiles === 0) { await new Promise<void>(done => setTimeout(done, 0)); options.signal?.throwIfAborted(); }
         activeRelPaths.add(relPath);
 
         let stat;
@@ -120,12 +193,13 @@ export class IngestionPipeline {
           continue;
         }
 
-        const mtimeMs = Math.round(stat.mtimeMs);
+        const mtimeMs = stat.mtimeMs;
         const sizeBytes = stat.size;
         const existingFile = existingFilesMap.get(relPath);
 
         // Tier 1: Fast OS metadata check (inode mtime + size + domain match)
         if (
+          !options.verifyContent &&
           existingFile &&
           existingFile.domain_id === domainId &&
           existingFile.mtime_ms === mtimeMs &&
@@ -136,7 +210,24 @@ export class IngestionPipeline {
         }
 
         // Tier 2: Content Hash verification
-        const content = readFileSync(fullPath, "utf-8");
+        if (sizeBytes > RESOURCE_LIMITS.sourceBytes) {
+          skippedFiles.push({path: relPath, reason: "file_too_large"});
+          if (existingFile) removedFiles++;
+          this.repo.deleteFileByPath(relPath);
+          this.repo.setMeta("slices_dirty", "1");
+          continue;
+        }
+        if (pendingUpdates.length >= RESOURCE_LIMITS.batchFiles || (batchBytes + sizeBytes > RESOURCE_LIMITS.batchBytes && pendingUpdates.length > 0)) { commitBatch(); await new Promise<void>(done => setTimeout(done, 0)); options.signal?.throwIfAborted(); }
+        let content: string;
+        try { content = readSourceFile(fullPath); }
+        catch (error) {
+          if (!(error instanceof SourceFileLimitError)) throw error;
+          skippedFiles.push({path: relPath, reason: "file_too_large"});
+          if (existingFile) removedFiles++;
+          this.repo.deleteFileByPath(relPath);
+          this.repo.setMeta("slices_dirty", "1");
+          continue;
+        }
         const currentHash = computeContentHash(content);
 
         if (
@@ -150,11 +241,19 @@ export class IngestionPipeline {
           continue;
         }
 
-        filesUpdated++;
         const lineCount = content.split("\n").length;
         const archetype = detectArchetype(relPath, domainCfg.archetypes ?? {});
         const ast = await this.parserEngine.parseFile(fullPath, content);
+        if (ast.status === "unsupported") {
+          unsupportedFiles.push(relPath);
+          this.repo.deleteFileByPath(relPath);
+          this.repo.setMeta("slices_dirty", "1");
+          continue;
+        }
+        if (ast.status === "degraded") throw new Error(`Cannot reliably index '${relPath}': parser is degraded.`);
 
+        filesUpdated++;
+        batchBytes += sizeBytes;
         pendingUpdates.push({
           fullPath,
           relPath,
@@ -167,33 +266,13 @@ export class IngestionPipeline {
         });
       }
 
-      // Batch ACID Transaction per domain
+      commitBatch();
       this.repo.runInTransaction(() => {
-        for (const meta of pendingMetadataUpdates) {
-          this.repo.updateFileMetadataOnly(meta.id, meta.mtimeMs, meta.sizeBytes);
-        }
-
-        for (const item of pendingUpdates) {
-          const fileId = this.repo.upsertFile(
-            domainId,
-            item.relPath,
-            item.archetype,
-            item.contentHash,
-            item.lineCount,
-            item.mtimeMs,
-            item.sizeBytes
-          );
-
-          this.repo.replaceFileSymbols(fileId, item.ast.symbols);
-          this.repo.replaceFileDependencies(fileId, item.ast.dependencies);
-
-          symbolsIndexed += item.ast.symbols.length;
-          dependenciesIndexed += item.ast.dependencies.length;
-        }
-
         // State Reconciliation: Prune deleted files from DB using pre-loaded existingFilesMap
         for (const [existingPath, existingRecord] of existingFilesMap.entries()) {
           if (existingRecord.domain_id === domainId && !activeRelPaths.has(existingPath)) {
+            removedFiles++;
+            this.repo.setMeta("slices_dirty", "1");
             this.repo.deleteFileByPath(existingPath);
             existingFilesMap.delete(existingPath);
           }
@@ -201,36 +280,78 @@ export class IngestionPipeline {
       });
     }
 
-    // Framework-Agnostic Semantic Vertical Slice Extraction (via Registry Strategy)
-    const detectedTopology = TopologyDetector.detect(projectRoot);
-    const semanticSlices = await SemanticSliceExtractorRegistry.extractAllSlices(
-      projectRoot,
-      detectedTopology.framework
-    );
-
-    const slicesToPersist =
-      semanticSlices.length > 0
-        ? semanticSlices
-        : new CallGraphTracer(this.repo, projectRoot).traceAllSlices();
-
-    this.repo.runInTransaction(() => {
-      this.repo.clearVerticalSlices();
-      for (const slice of slicesToPersist) {
-        this.repo.upsertVerticalSlice(slice);
-      }
-    });
+    options.signal?.throwIfAborted();
+    const topologyInputs = ["package.json", "composer.json", "nest-cli.json", "go.mod", "artisan", "tsconfig.json"]
+      .map(file => { const target = join(projectRoot, file); return existsSync(target) ? readSourceFile(target) : ""; });
+    const semanticRevision = computeContentHash(JSON.stringify([config.domains, topologyInputs]));
+    // Membership covers newly added semantic files outside registered domains.
+    const inventory = this.collectFiles(projectRoot, projectRoot, config.settings.ignore_patterns, [], options.signal).map(({fullPath, relPath}) => {
+      const stat = statSync(fullPath);
+      const fingerprint = `${stat.size}:${stat.mtimeMs}`;
+      return `${relPath}:${fingerprint}`;
+    }).sort();
+    const inventoryRevision = computeContentHash(JSON.stringify(inventory));
+    const priorInputs: SourceReadFingerprint[] = JSON.parse(this.repo.getMeta("semantic_inputs") || "[]");
+    let semanticInputChanged = false;
+    for (let index = 0; index < priorInputs.length; index++) {
+      if (index % RESOURCE_LIMITS.batchFiles === 0) { await new Promise<void>(done => setTimeout(done, 0)); options.signal?.throwIfAborted(); }
+      const input = priorInputs[index];
+      const file = resolve(projectRoot, input.path);
+      if (!existsSync(file)) { semanticInputChanged = true; break; }
+      const stat = statSync(file);
+      if (stat.size !== input.size || stat.mtimeMs !== input.mtime) { semanticInputChanged = true; break; }
+      if (!options.verifyContent) continue;
+      try { semanticInputChanged = computeContentHash(readSourceFile(file)) !== input.hash; }
+      catch (error) { if (error instanceof SourceFileLimitError) semanticInputChanged = input.hash !== "file_too_large"; else throw error; }
+      if (semanticInputChanged) break;
+    }
+    const extractorRevision = "1"; // Bump when extractor or inferred-chain semantics change.
+    if (this.repo.getMeta("semantic_extractor_revision") !== extractorRevision || this.repo.getMeta("semantic_inventory") !== inventoryRevision || semanticInputChanged || filesUpdated > 0 || removedFiles > 0 || this.repo.getMeta("slices_dirty") === "1" || this.repo.getMeta("semantic_revision") !== semanticRevision) {
+      const extracted = await collectSourceSkips(projectRoot, async () => {
+        const detectedTopology = TopologyDetector.detect(projectRoot);
+        const semanticSlices = await SemanticSliceExtractorRegistry.extractAllSlices(projectRoot, detectedTopology.framework);
+        return semanticSlices.length > 0 ? semanticSlices : new CallGraphTracer(this.repo, projectRoot).traceAllSlices();
+      });
+      skippedFiles.push(...extracted.skipped);
+      this.repo.runInTransaction(() => {
+        this.repo.clearVerticalSlices();
+        for (const slice of extracted.value) this.repo.upsertVerticalSlice(slice);
+        this.repo.setMeta("slices_dirty", "");
+        this.repo.setMeta("semantic_revision", semanticRevision);
+        this.repo.setMeta("semantic_extractor_revision", extractorRevision);
+        this.repo.setMeta("semantic_inventory", inventoryRevision);
+        this.repo.setMeta("semantic_inputs", JSON.stringify(extracted.inputs));
+        this.repo.setMeta("semantic_skipped_files", JSON.stringify(extracted.skipped));
+      });
+    }
 
+    skippedFiles.push(...JSON.parse(this.repo.getMeta("semantic_skipped_files") || "[]"));
     const durationMs = Math.round(performance.now() - startTime);
 
+    // Identify uncovered eligible source files outside registered domains
+    const indexedPaths = new Set(this.repo.getAllFiles().map((file) => file.path));
+    const allWorkspaceFiles = this.collectFiles(projectRoot, projectRoot, config.settings.ignore_patterns, [], options.signal);
+    const unsupportedSet = new Set(unsupportedFiles);
+    const skippedSet = new Set(skippedFiles.map((file) => file.path));
+    const uncoveredFiles = allWorkspaceFiles
+      .map((file) => file.relPath)
+      .filter((rel) => !indexedPaths.has(rel) && !unsupportedSet.has(rel) && !skippedSet.has(rel));
+
     // Update SSOT repo_meta watermark
     this.repo.runInTransaction(() => {
       this.repo.setMeta("last_sync", new Date().toISOString());
+      this.repo.setMeta("sync_duration_ms", String(durationMs));
       this.repo.setMeta("files_scanned", String(filesScanned));
       this.repo.setMeta("files_updated", String(filesUpdated));
       this.repo.setMeta("symbols_indexed", String(symbolsIndexed));
       this.repo.setMeta("dependencies_indexed", String(dependenciesIndexed));
+      this.repo.setMeta("unsupported_files", JSON.stringify(unsupportedFiles));
+      this.repo.setMeta("skipped_files", JSON.stringify([...new Map(skippedFiles.map(file => [file.path, file])).values()]));
+      this.repo.setMeta("uncovered_files", JSON.stringify(uncoveredFiles));
     });
 
+    this.repo.checkpoint();
+
     return {
       domains_processed: domainNames.length,
       files_scanned: filesScanned,
@@ -419,23 +540,43 @@ export class IngestionPipeline {
     dir: string,
     workspaceRoot: string,
     ignorePatterns: string[],
-    childDomainRoots: string[] = []
+    childDomainRoots: string[] = [],
+    signal?: AbortSignal
   ): Array<{ fullPath: string; relPath: string }> {
     const results: Array<{ fullPath: string; relPath: string }> = [];
 
+    // SV-02: Check if initial dir itself is an escaping symlink
+    try {
+      const dirLstat = lstatSync(dir);
+      if (dirLstat.isSymbolicLink()) {
+        const real = realpathSync(dir);
+        if (!isPathContained(real, workspaceRoot)) {
+          throw new Error(`Directory '${dir}' is a symlink escaping workspace boundary '${workspaceRoot}'.`);
+        }
+      }
+    } catch (err: any) {
+      if (err?.code !== "ENOENT") throw err;
+    }
+
     const walk = (currentDir: string) => {
+      signal?.throwIfAborted();
       let entries: string[] = [];
       try {
         entries = readdirSync(currentDir);
-      } catch {
-        return;
+      } catch (err: any) {
+        // SV-01: Do not swallow permission/IO errors; propagate so pipeline reports incomplete/error
+        if (err?.code === "ENOENT") {
+          return;
+        }
+        throw new Error(`Failed to read directory '${currentDir}': ${err.message || err}`);
       }
 
       for (const entry of entries) {
+        signal?.throwIfAborted();
         const fullPath = join(currentDir, entry);
         const relPath = relative(workspaceRoot, fullPath).replace(/\\/g, "/");
 
-        if (this.shouldIgnore(relPath, ignorePatterns)) {
+        if (shouldIgnoreSourcePath(relPath, ignorePatterns)) {
           continue;
         }
 
@@ -444,14 +585,35 @@ export class IngestionPipeline {
         }
 
         try {
-          const stat = statSync(fullPath);
+          const stat = lstatSync(fullPath);
+          if (stat.isSymbolicLink()) {
+            try {
+              const real = realpathSync(fullPath);
+              if (!isPathContained(real, workspaceRoot)) {
+                continue;
+              }
+              const targetStat = statSync(real);
+              if (targetStat.isDirectory()) {
+                continue;
+              } else if (targetStat.isFile() && this.isSourceFile(real)) {
+                results.push({ fullPath, relPath });
+              }
+            } catch {
+              continue;
+            }
+            continue;
+          }
+
           if (stat.isDirectory()) {
             walk(fullPath);
           } else if (stat.isFile() && this.isSourceFile(fullPath)) {
             results.push({ fullPath, relPath });
           }
-        } catch {
-          // ignore unreadable files
+        } catch (err: any) {
+          if (err?.code === "ENOENT") {
+            continue;
+          }
+          throw new Error(`Failed to stat path '${fullPath}': ${err.message || err}`);
         }
       }
     };
@@ -471,58 +633,37 @@ export class IngestionPipeline {
     if (config.settings.db_path && config.settings.db_path !== ":memory:") {
       const dbDir = dirname(resolve(config.settings.db_path));
       const candidate = basename(dbDir) === ".septum" ? dirname(dbDir) : dbDir;
-      const absDomainRoot = resolve(domainRoot);
+      const absDomainRoot = isAbsolute(domainRoot) ? domainRoot : resolve(candidate, domainRoot);
       if (absDomainRoot.startsWith(candidate)) {
         return candidate;
       }
     }
-    return process.cwd();
+    return resolveWorkspaceRoot();
   }
 
-  private shouldIgnore(filePath: string, ignorePatterns: string[]): boolean {
-    const normPath = filePath.replace(/\\/g, "/").replace(/^\.\//, "");
-    for (const pattern of ignorePatterns) {
-      const normPattern = pattern.replace(/\\/g, "/").replace(/^\.\//, "").trim();
-      if (!normPattern) continue;
-
-      if (normPattern === "*" || normPattern === "**") {
-        return true;
-      }
-
-      if (normPath === normPattern) return true;
-
-      if (normPattern.endsWith("/**")) {
-        const prefix = normPattern.slice(0, -3);
-        if (normPath === prefix || normPath.startsWith(prefix + "/")) return true;
-      } else if (normPattern.endsWith("/*")) {
-        const prefix = normPattern.slice(0, -2);
-        if (normPath === prefix || normPath.startsWith(prefix + "/")) return true;
-      } else if (normPath.startsWith(normPattern + "/")) {
-        return true;
-      }
-
-      if (normPattern.startsWith("*.") && normPath.endsWith(normPattern.slice(1))) {
-        return true;
-      }
 
-      const segments = normPath.split("/");
-      if (segments.includes(normPattern)) {
-        return true;
-      }
-    }
-    return false;
+  private isSourceFile(path: string): boolean {
+    return ASTParserEngine.isSourceFile(path);
   }
 
-  private isSourceFile(path: string): boolean {
-    return /\.(ts|tsx|js|jsx|mjs|cjs|php|py|go|rs|vue|astro|svelte)$/i.test(path);
+  private ensureParserRevision(): void {
+    const revision = "2";
+    if (this.repo.getMeta("parser_revision") === revision) return;
+    this.repo.runInTransaction(() => {
+      for (const file of this.repo.getAllFiles()) {
+        this.repo.upsertFile(file.domain_id, file.path, file.archetype, "", file.line_count, -1, -1);
+      }
+      this.repo.clearVerticalSlices();
+      this.repo.setMeta("parser_revision", revision);
+    });
   }
 
   /**
    * JIT single-file ingestion: quickly verifies and updates a single file's AST,
    * symbols, and dependencies in SQLite if its mtime or content hash has changed.
-   * Runs in < 5ms for rapid delta sync without full project scans.
    */
   public async ingestFile(targetFilePath: string, config: ValidatedSeptumConfig): Promise<boolean> {
+    this.ensureParserRevision();
     const fullPath = resolve(targetFilePath);
     if (!existsSync(fullPath)) return false;
 
@@ -534,28 +675,18 @@ export class IngestionPipeline {
       return false;
     }
 
-    const mtimeMs = Math.round(stat.mtimeMs);
+    const mtimeMs = stat.mtimeMs;
     const sizeBytes = stat.size;
 
-    const projectRoot = this.customWorkspaceRoot || process.cwd();
+    const projectRoot = this.customWorkspaceRoot || resolveWorkspaceRoot();
     const relPath = relative(projectRoot, fullPath).replace(/\\/g, "/");
 
-    if (this.shouldIgnore(relPath, config.settings.ignore_patterns ?? [])) {
+    if (shouldIgnoreSourcePath(relPath, config.settings.ignore_patterns ?? [])) {
       return false;
     }
 
-    let targetDomainName: string | null = null;
-    let targetDomainCfg = null;
-
-    for (const [dName, dCfg] of Object.entries(config.domains)) {
-      const absRoot = resolve(projectRoot, dCfg.root).replace(/\\/g, "/");
-      const normFullPath = fullPath.replace(/\\/g, "/");
-      if (normFullPath === absRoot || normFullPath.startsWith(absRoot + "/")) {
-        targetDomainName = dName;
-        targetDomainCfg = dCfg;
-        break;
-      }
-    }
+    const targetDomainName = new ModuleResolver(projectRoot, this.repo).resolveSourceDomain(fullPath, config);
+    const targetDomainCfg = targetDomainName ? config.domains[targetDomainName] : null;
 
     if (!targetDomainName || !targetDomainCfg) {
       return false;
@@ -564,20 +695,23 @@ export class IngestionPipeline {
     const domainId = this.repo.upsertDomain(targetDomainName, targetDomainCfg);
     const existingFile = this.repo.getFileByPath(relPath);
 
-    // Tier 1: Fast OS metadata check
-    if (
-      existingFile &&
-      existingFile.mtime_ms === mtimeMs &&
-      existingFile.size_bytes === sizeBytes
-    ) {
-      return false; // Completely up to date
+    let content: string;
+    try { content = readSourceFile(fullPath); }
+    catch (error) {
+      if (!(error instanceof SourceFileLimitError)) throw error;
+      this.repo.runInTransaction(() => {
+        this.repo.deleteFileByPath(relPath);
+        this.repo.clearVerticalSlices();
+        this.repo.setMeta("slices_dirty", "1");
+        const skipped: SkippedSourceFile[] = JSON.parse(this.repo.getMeta("skipped_files") || "[]");
+        this.repo.setMeta("skipped_files", JSON.stringify([...skipped.filter(file => file.path !== relPath), {path: relPath, reason: "file_too_large"}]));
+      });
+      return false;
     }
-
-    const content = readFileSync(fullPath, "utf-8");
     const currentHash = computeContentHash(content);
 
     // Tier 2: Hash verification
-    if (existingFile && existingFile.content_hash === currentHash) {
+    if (existingFile && existingFile.domain_id === domainId && existingFile.content_hash === currentHash) {
       this.repo.updateFileMetadataOnly(existingFile.id, mtimeMs, sizeBytes);
       return false;
     }
@@ -585,6 +719,14 @@ export class IngestionPipeline {
     const lineCount = content.split("\n").length;
     const archetype = detectArchetype(relPath, targetDomainCfg.archetypes ?? {});
     const ast = await this.parserEngine.parseFile(fullPath, content);
+    if (ast.status === "unsupported") {
+      this.repo.deleteFileByPath(relPath);
+      const unsupported = new Set<string>(JSON.parse(this.repo.getMeta("unsupported_files") || "[]"));
+      unsupported.add(relPath);
+      this.repo.setMeta("unsupported_files", JSON.stringify([...unsupported]));
+      return false;
+    }
+    if (ast.status === "degraded") throw new Error(`Cannot reliably index '${relPath}': parser is degraded.`);
 
     this.repo.runInTransaction(() => {
       const fileId = this.repo.upsertFile(
@@ -598,6 +740,12 @@ export class IngestionPipeline {
       );
       this.repo.replaceFileSymbols(fileId, ast.symbols);
       this.repo.replaceFileDependencies(fileId, ast.dependencies);
+      this.repo.clearVerticalSlices();
+      this.repo.setMeta("slices_dirty", "1");
+      const skipped: SkippedSourceFile[] = JSON.parse(this.repo.getMeta("skipped_files") || "[]");
+      this.repo.setMeta("skipped_files", JSON.stringify(skipped.filter(file => file.path !== relPath)));
+      const unsupported = JSON.parse(this.repo.getMeta("unsupported_files") || "[]") as string[];
+      this.repo.setMeta("unsupported_files", JSON.stringify(unsupported.filter(path => path !== relPath)));
     });
 
     return true;

package/src/core/parser/extractors/frontend.ts

@@ -123,6 +123,7 @@ export class FrontendExtractor implements CodeExtractor {
     dependencies: ExtractedDependency[]
   ): Promise<void> {
     const parsed = await this.tsExtractor.extract("component-script.ts", scriptContent);
+    if (parsed.status === "degraded") throw new Error("Cannot reliably parse component script.");
     const delta = Math.max(0, startLine - 1);
 
     for (const dep of parsed.dependencies) {

package/src/core/parser/extractors/go.ts

@@ -67,10 +67,10 @@ export class GoExtractor implements CodeExtractor {
       this.extractFromAst(rootNode, filePath, content)
     );
     if (astResult) {
-      return astResult;
+      return { ...astResult, status: "parsed" };
     }
 
-    return this.extractFallback(filePath, content);
+    return { ...this.extractFallback(filePath, content), status: "degraded" };
   }
 
   private extractFallback(filePath: string, content: string): ParsedFileAST {