seer-mcp 0.1.0

package/dist/indexer/index.js

@@ -0,0 +1,930 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Indexer = void 0;
+const fs_1 = __importDefault(require("fs"));
+const crypto_1 = __importDefault(require("crypto"));
+const path_1 = __importDefault(require("path"));
+const discovery_js_1 = require("./discovery.js");
+const index_js_1 = require("../parser/index.js");
+const workerpool_js_1 = require("../parser/workerpool.js");
+const pagerank_js_1 = require("../graph/pagerank.js");
+const classify_js_1 = require("./classify.js");
+const modules_js_1 = require("./modules.js");
+const boundaries_js_1 = require("./boundaries.js");
+const shapehash_js_1 = require("./shapehash.js");
+const continuity_js_1 = require("./continuity.js");
+const serviceLinks_js_1 = require("./serviceLinks.js");
+const protoScanner_js_1 = require("./protoScanner.js");
+const serviceHostScanner_js_1 = require("./serviceHostScanner.js");
+const DEFAULT_MAX_FILE_BYTES = 0; // no cap by default — completeness first
+const DEFAULT_IO_CONCURRENCY = 8; // matches the file-handle budget on most OSes comfortably
+const DEFAULT_IO_PREFETCH_BYTES = 64 * 1024 * 1024; // 64 MiB
+const PARALLEL_AUTO_MIN_FILES = 100; // below this, default to serial unless explicitly forced
+// Filenames that are almost always generated boilerplate (Unreal Header Tool
+// produces *.generated.h; protobufs produce *.pb.h / *.pb.cc; etc.). We skip
+// them at the per-file level so the discovery glob can stay simple.
+const SKIP_FILENAME_PATTERNS = [
+    /\.generated\.h$/i,
+    /\.gen\.cpp$/i,
+    /\.gen\.h$/i,
+    /\.pb\.cc$/,
+    /\.pb\.h$/,
+];
+function shouldSkipFilename(relativePath) {
+    return SKIP_FILENAME_PATTERNS.some(re => re.test(relativePath));
+}
+// ── Byte-aware semaphore ────────────────────────────────────────────────────────
+//
+// Caps total bytes of file content held in the prefetch buffer. A waiter is
+// admitted as soon as either (a) the new total fits within `capacity`, or
+// (b) the budget is empty (so a single oversize file never deadlocks — we
+// always allow at least one read in flight).
+//
+// The implementation is deliberately FIFO: we only wake the *head* waiter, so
+// a flood of small reads can't perpetually starve a single large one queued
+// behind them.
+class ByteSemaphore {
+    capacity;
+    bytes = 0;
+    waiters = [];
+    constructor(capacity) {
+        this.capacity = capacity;
+    }
+    async acquire(requested) {
+        if (this.bytes === 0 || this.bytes + requested <= this.capacity) {
+            this.bytes += requested;
+            return;
+        }
+        await new Promise(resolve => {
+            this.waiters.push({ bytes: requested, resolve });
+        });
+        // `release` has already added `requested` to `this.bytes` on our behalf
+        // before resolving — see `release()` below.
+    }
+    release(returned) {
+        this.bytes -= returned;
+        if (this.bytes < 0)
+            this.bytes = 0; // defensive: never go negative
+        while (this.waiters.length > 0) {
+            const next = this.waiters[0];
+            if (this.bytes === 0 || this.bytes + next.bytes <= this.capacity) {
+                this.bytes += next.bytes;
+                this.waiters.shift();
+                next.resolve();
+            }
+            else {
+                break;
+            }
+        }
+    }
+}
+class Indexer {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    async indexDirectory(repoRoot, options = {}) {
+        const start = Date.now();
+        const absRoot = path_1.default.resolve(repoRoot);
+        const quiet = options.quiet && !options.verbose;
+        if (options.verbose) {
+            process.stdout.write(`\nDiscovering files in ${absRoot}...\n`);
+        }
+        const files = await (0, discovery_js_1.discoverFiles)(absRoot, {
+            includeVendor: options.includeVendor,
+            includeGenerated: options.includeGenerated,
+            mode: options.mode,
+        });
+        const total = files.length;
+        const maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
+        const ioConcurrency = Math.max(1, options.ioConcurrency ?? DEFAULT_IO_CONCURRENCY);
+        const ioPrefetchBytes = Math.max(1, options.ioPrefetchBytes ?? DEFAULT_IO_PREFETCH_BYTES);
+        // Track every file_id we touch this run so we can prune ones left over
+        // from a previous run (e.g. files now hidden by a new ignore rule).
+        const touchedFileIds = new Set();
+        let indexed = 0;
+        let reusedFromCache = 0;
+        let skipped = 0;
+        let skippedTooLarge = 0;
+        let parseErrors = 0;
+        let workerWasmResets = 0;
+        // ── Pre-filter into a work queue ────────────────────────────────────────────
+        // Pure CPU work (string ops on the path). Cheap to do all at once so the
+        // async prefetcher's index space matches up cleanly with progress counters.
+        const work = [];
+        for (const file of files) {
+            const language = (0, index_js_1.detectLanguage)(file.absolutePath);
+            if (!language || shouldSkipFilename(file.relativePath)) {
+                skipped++;
+                continue;
+            }
+            work.push({ file, language });
+        }
+        // Existing pre-v8 DBs can have all source hashes cached but no
+        // service_calls rows yet. Force one full parse pass so Track-G evidence is
+        // backfilled, then mark completion in finishIndex().
+        const forceServiceCallBackfill = this.store.needsServiceCallBackfill();
+        // ── Batched transactions ──────────────────────────────────────────────────
+        // The Phase-1 design wrapped each file's inserts in its own SQLite
+        // transaction. That works but every commit fsyncs the WAL, which adds
+        // O(milliseconds) of overhead per file. For 40k+ file repos the commit
+        // overhead dominates the per-file budget. Batching N files per transaction
+        // amortizes the fsync cost N-fold.
+        //
+        // Trade-off: a fatal error mid-batch rolls back at most BATCH_SIZE files'
+        // worth of inserts (which we'd just re-do on the next run). We never lose
+        // user data — only re-do work — so a moderately large batch is safe.
+        //
+        // The hash-skip path participates in the batch too (its UPDATE indexed_at
+        // and DELETE-old-symbols statements were previously running as
+        // autocommitted singletons — now they share a transaction with the file's
+        // inserts).
+        const BATCH_SIZE = 200;
+        let batchOpen = false;
+        const openBatch = () => {
+            if (!batchOpen) {
+                this.store.begin();
+                batchOpen = true;
+            }
+        };
+        const closeBatch = () => {
+            if (batchOpen) {
+                this.store.commit();
+                batchOpen = false;
+            }
+        };
+        const rollbackBatch = () => {
+            if (batchOpen) {
+                try {
+                    this.store.rollback();
+                }
+                catch { /* best effort */ }
+                batchOpen = false;
+            }
+        };
+        if (!options.verbose && !quiet) {
+            writeProgress(0, total, '');
+        }
+        // ── Parallel-parsing branch (worker pool) ───────────────────────────────────
+        //
+        // When enabled, each file's read + hash + parse runs in a worker_threads
+        // worker (its own WASM heap). The pool delivers results to the callback
+        // STRICTLY in input order, so symbol-id insertion order — and therefore
+        // every cross-run-stable scale-test invariant — is identical to the
+        // serial path. DB writes still run single-writer on the main thread.
+        //
+        // Result-kind contract (matches the serial branch's semantics exactly):
+        //   parsed     → upsertFileWithCache → touchedFileIds.add → insert all
+        //   parse-error → upsertFileWithCache → touchedFileIds.add → no inserts
+        //   cached     → upsertFileWithCache → touchedFileIds.add → no inserts
+        //                  (worker confirmed hash === expectedHash; upsert sees
+        //                   the same hash and returns unchanged=true)
+        //   too-large  → counter only; file row NOT touched → pruned
+        //   io-error   → counter only; file row NOT touched → pruned
+        //
+        // The cached/parse-error upsert calls are CRITICAL: without them
+        // `touchedFileIds` would not contain those file ids and
+        // `pruneFilesNotIn(touchedFileIds)` below would delete every unchanged
+        // cached file from the DB.
+        // Auto-enabled for normal/large workspaces. Tiny workspaces stay serial by
+        // default to avoid worker startup/churn; force workers with `parallel: true`
+        // or `SEER_PARALLEL_PARSE=1`. Opt out with `parallel: false` or
+        // `SEER_PARALLEL_PARSE=0`.
+        const envParallel = typeof process !== 'undefined' && process.env != null
+            ? (process.env.SEER_PARALLEL_PARSE === '0' ? false
+                : process.env.SEER_PARALLEL_PARSE === '1' ? true
+                    : undefined)
+            : undefined;
+        const parallelRequested = options.parallel ?? envParallel ?? true;
+        const parallelForced = options.parallel === true || envParallel === true;
+        const parallelEnabled = parallelRequested && (parallelForced || work.length >= PARALLEL_AUTO_MIN_FILES);
+        if (parallelEnabled && work.length > 0) {
+            // Snapshot known DB hashes so workers can skip parsing on cache hits.
+            const cacheMap = new Map();
+            for (const f of this.store.listFiles())
+                cacheMap.set(f.path, f.hash);
+            const poolItems = work.map(w => ({
+                abs: w.file.absolutePath,
+                lang: w.language,
+                expectedHash: forceServiceCallBackfill ? null : cacheMap.get(w.file.absolutePath) ?? null,
+                maxFileBytes,
+            }));
+            const pool = new workerpool_js_1.WorkerPool({ jobs: options.jobs });
+            try {
+                await pool.ready();
+                let processed = 0;
+                await pool.dispatch(poolItems, (seq, result) => {
+                    processed++;
+                    const w = work[seq];
+                    const rel = w.file.relativePath;
+                    // Counters-only branches (file row stays untouched → pruned).
+                    if (result.kind === 'too-large') {
+                        skippedTooLarge++;
+                        if (options.verbose) {
+                            process.stdout.write(`  ⤬  ${rel} (${(result.size / 1024).toFixed(0)} KiB > ${(maxFileBytes / 1024).toFixed(0)} KiB cap)\n`);
+                        }
+                        else if (!quiet)
+                            writeProgress(processed, total, rel);
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        return;
+                    }
+                    if (result.kind === 'io-error') {
+                        skipped++;
+                        if (options.verbose)
+                            process.stdout.write(`  ⚠  ${rel} (read error: ${result.error})\n`);
+                        else if (!quiet)
+                            writeProgress(processed, total, rel);
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        return;
+                    }
+                    // parsed / parse-error / cached all read the file successfully —
+                    // we have hash + lines. Always upsert so touchedFileIds is updated.
+                    const hash = result.hash;
+                    const lines = result.lines;
+                    openBatch();
+                    const classification = (0, classify_js_1.classifyFile)(rel);
+                    const upserted = forceServiceCallBackfill
+                        ? { fileId: this.store.upsertFile(w.file.absolutePath, rel, w.language, hash, lines, classification), unchanged: false }
+                        : this.store.upsertFileWithCache(w.file.absolutePath, rel, w.language, hash, lines, classification);
+                    const { fileId, unchanged } = upserted;
+                    touchedFileIds.add(fileId);
+                    // Cache hit (worker's hash matched the DB's stored hash). Prior
+                    // symbols/edges/imports/routes/configKeys stay as-is. Note: an
+                    // explicit `cached` result always falls into this branch; a `parsed`
+                    // result whose hash happens to match an in-flight DB update would
+                    // also land here defensively (we never re-insert when unchanged).
+                    if (unchanged) {
+                        reusedFromCache++;
+                        if (options.verbose)
+                            process.stdout.write(`  =  ${rel} (cached)\n`);
+                        else if (!quiet)
+                            writeProgress(processed, total, rel);
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        return;
+                    }
+                    // Only `parsed` carries an extraction. `cached` lands in the
+                    // unchanged-branch above; `parse-error` and any defensive fall-
+                    // through here get treated as a parse error (file row exists,
+                    // no symbols/edges emitted).
+                    if (result.kind !== 'parsed') {
+                        parseErrors++;
+                        if (options.verbose)
+                            process.stdout.write(`  ⚠  ${rel} (parse error)\n`);
+                        else if (!quiet)
+                            writeProgress(processed, total, rel);
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        return;
+                    }
+                    // parsed: insert all symbols, edges, imports, routes, configKeys.
+                    const extraction = result.extraction;
+                    const symbolIdMap = new Map();
+                    for (const def of extraction.definitions) {
+                        const symId = this.store.insertSymbol(fileId, def);
+                        const qname = def.qualifiedName ?? def.name;
+                        if (!symbolIdMap.has(qname))
+                            symbolIdMap.set(qname, symId);
+                    }
+                    for (const ref of extraction.references) {
+                        const fromId = ref.callerName ? symbolIdMap.get(ref.callerName) : undefined;
+                        if (fromId !== undefined) {
+                            this.store.insertEdge(fromId, ref.calleeName, ref.kind, ref.line);
+                        }
+                    }
+                    for (const mod of extraction.importedModules) {
+                        this.store.insertFileImport(fileId, mod);
+                    }
+                    if (extraction.routes) {
+                        for (const r of extraction.routes) {
+                            this.store.insertRoute(fileId, r.method, r.path, r.framework, r.handlerName ?? null, r.line, {
+                                protocol: r.protocol ?? 'http',
+                                operation: r.operation ?? null,
+                                topic: r.topic ?? null,
+                                queue: r.queue ?? null,
+                                exchange: r.exchange ?? null,
+                                service: r.service ?? null,
+                                broker: r.broker ?? null,
+                                metadataJson: r.metadataJson ?? null,
+                            });
+                        }
+                    }
+                    if (extraction.configKeys) {
+                        for (const c of extraction.configKeys) {
+                            const enclosingId = c.callerName ? symbolIdMap.get(c.callerName) ?? null : null;
+                            this.store.insertConfigKey(c.key, c.source, fileId, enclosingId, c.line);
+                        }
+                    }
+                    if (extraction.serviceCalls) {
+                        for (const sc of extraction.serviceCalls) {
+                            const enclosingId = sc.callerName ? symbolIdMap.get(sc.callerName) ?? null : null;
+                            // Only run HTTP-shaped normalization when the call is HTTP.
+                            const norm = sc.protocol === 'http'
+                                ? (0, serviceLinks_js_1.normalizeHttpTarget)(sc.rawTarget)
+                                : { path: undefined, hostHint: undefined };
+                            this.store.insertServiceCall({
+                                fileId,
+                                symbolId: enclosingId,
+                                protocol: sc.protocol,
+                                method: sc.method ?? null,
+                                rawTarget: sc.rawTarget,
+                                normalizedPath: sc.normalizedPath ?? norm.path ?? null,
+                                hostHint: sc.hostHint ?? norm.hostHint ?? null,
+                                envKey: sc.envKey ?? null,
+                                framework: sc.framework,
+                                line: sc.line,
+                                confidence: sc.confidence,
+                                operation: sc.operation ?? null,
+                                topic: sc.topic ?? null,
+                                queue: sc.queue ?? null,
+                                exchange: sc.exchange ?? null,
+                                service: sc.service ?? null,
+                                broker: sc.broker ?? null,
+                                metadataJson: sc.metadataJson ?? null,
+                            });
+                        }
+                    }
+                    if (processed % BATCH_SIZE === 0)
+                        closeBatch();
+                    indexed++;
+                    if (options.verbose) {
+                        process.stdout.write(`  ✓  ${rel} (${extraction.definitions.length} symbols, ${extraction.references.length} refs)\n`);
+                    }
+                    else if (!quiet) {
+                        writeProgress(processed, total, rel);
+                    }
+                });
+                workerWasmResets = pool.wasmResetCount();
+                closeBatch();
+            }
+            catch (err) {
+                rollbackBatch();
+                await pool.terminate().catch(() => { });
+                throw err;
+            }
+            await pool.shutdown();
+            if (!options.verbose && !quiet)
+                process.stdout.write('\n');
+            // Skip the serial prefetcher block below.
+            return await this.finishIndex(absRoot, start, total, indexed, reusedFromCache, skipped, skippedTooLarge, parseErrors, touchedFileIds, { verbose: options.verbose, quiet: !!quiet, workerWasmResets });
+        }
+        // ── Bounded async prefetcher (serial branch) ────────────────────────────────
+        //
+        // Producer side: a fixed sliding window of up to `ioConcurrency` in-flight
+        // `prefetchOne()` calls, each one bounded by `byteSem` so cumulative
+        // buffered content never exceeds `ioPrefetchBytes`.
+        //
+        // Consumer side: the main loop awaits prefetched results IN ORDER (so
+        // batching/progress/determinism match the old sync loop exactly), parses
+        // serially (single shared WASM module — see parser/index.ts), and writes
+        // serially (single SQLite connection).
+        //
+        // The byte budget is released only AFTER parse completes, because parse
+        // reads from the in-memory string. That means while a slow file parses we
+        // hold its budget — which is intentional: backpressures the prefetcher
+        // exactly when the parser falls behind.
+        const byteSem = new ByteSemaphore(ioPrefetchBytes);
+        const prefetchOne = async (idx) => {
+            const item = work[idx];
+            const abs = item.file.absolutePath;
+            // Only stat when there's an actual size cap to enforce. Saves one
+            // syscall per file in the (default) `maxFileBytes === 0` mode.
+            if (maxFileBytes > 0) {
+                let size;
+                try {
+                    size = (await fs_1.default.promises.stat(abs)).size;
+                }
+                catch {
+                    return { kind: 'io-error', item };
+                }
+                if (size > maxFileBytes) {
+                    return { kind: 'too-large', item, size };
+                }
+                await byteSem.acquire(Math.max(size, 1));
+                let content;
+                try {
+                    content = await fs_1.default.promises.readFile(abs, 'utf8');
+                }
+                catch {
+                    byteSem.release(Math.max(size, 1));
+                    return { kind: 'io-error', item };
+                }
+                // Update the held budget if the on-disk size disagreed with the
+                // decoded string length (Buffer length, after UTF-8 → UTF-16). We
+                // re-anchor to the actual content length so future releases match.
+                const actual = Buffer.byteLength(content, 'utf8');
+                if (actual !== size) {
+                    byteSem.release(Math.max(size, 1));
+                    await byteSem.acquire(Math.max(actual, 1));
+                }
+                return { kind: 'ok', item, content, size: actual };
+            }
+            // No size cap → skip the stat entirely. We don't know the size up
+            // front, so reserve a conservative slot (1 byte), read, then re-acquire
+            // the true size. This keeps a single huge file from blocking us before
+            // we even know it's huge.
+            await byteSem.acquire(1);
+            let content;
+            try {
+                content = await fs_1.default.promises.readFile(abs, 'utf8');
+            }
+            catch {
+                byteSem.release(1);
+                return { kind: 'io-error', item };
+            }
+            const size = Buffer.byteLength(content, 'utf8');
+            // Re-anchor budget to actual size.
+            byteSem.release(1);
+            await byteSem.acquire(Math.max(size, 1));
+            return { kind: 'ok', item, content, size };
+        };
+        // Sliding window of in-flight prefetches, indexed by their position in `work`.
+        // We `slots.shift()` after awaiting so the array stays small (≤ ioConcurrency).
+        const slots = [];
+        let nextToLaunch = 0;
+        const launchUpTo = (window) => {
+            while (slots.length < window && nextToLaunch < work.length) {
+                slots.push(prefetchOne(nextToLaunch));
+                nextToLaunch++;
+            }
+        };
+        // Prime the pipeline.
+        launchUpTo(ioConcurrency);
+        let processed = 0;
+        try {
+            while (slots.length > 0) {
+                const prefetched = await slots.shift();
+                processed++;
+                // Whatever happens to this file, the moment we're done with the
+                // string we MUST release its byte budget so the next prefetch can
+                // start. We accumulate the released amount and release in `finally`
+                // at the end of each iteration.
+                const heldBytes = prefetched.kind === 'ok' ? Math.max(prefetched.size, 1) :
+                    prefetched.kind === 'too-large' ? 0 : // never acquired
+                        0; // io-error already released
+                try {
+                    if (prefetched.kind === 'too-large') {
+                        skippedTooLarge++;
+                        if (options.verbose) {
+                            process.stdout.write(`  ⤬  ${prefetched.item.file.relativePath} (${(prefetched.size / 1024).toFixed(0)} KiB > ${(maxFileBytes / 1024).toFixed(0)} KiB cap)\n`);
+                        }
+                        else if (!quiet) {
+                            writeProgress(processed, total, prefetched.item.file.relativePath);
+                        }
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        continue;
+                    }
+                    if (prefetched.kind === 'io-error') {
+                        skipped++;
+                        if (options.verbose) {
+                            process.stdout.write(`  ⚠  ${prefetched.item.file.relativePath} (read error)\n`);
+                        }
+                        else if (!quiet) {
+                            writeProgress(processed, total, prefetched.item.file.relativePath);
+                        }
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        continue;
+                    }
+                    const { item, content } = prefetched;
+                    const { file, language } = item;
+                    const hash = sha256(content);
+                    const lines = content.split('\n').length;
+                    openBatch();
+                    const classification = (0, classify_js_1.classifyFile)(file.relativePath);
+                    const upserted = forceServiceCallBackfill
+                        ? { fileId: this.store.upsertFile(file.absolutePath, file.relativePath, language, hash, lines, classification), unchanged: false }
+                        : this.store.upsertFileWithCache(file.absolutePath, file.relativePath, language, hash, lines, classification);
+                    const { fileId, unchanged } = upserted;
+                    touchedFileIds.add(fileId);
+                    // Hash-based cache hit: same content as last run → keep symbols, edges,
+                    // and file_imports as-is. Edge to_ids that point to symbols that have
+                    // since been deleted got NULLed by the FK cascade, so resolveEdges()
+                    // below will still re-link them.
+                    if (unchanged) {
+                        reusedFromCache++;
+                        if (options.verbose) {
+                            process.stdout.write(`  =  ${file.relativePath} (cached)\n`);
+                        }
+                        else if (!quiet) {
+                            writeProgress(processed, total, file.relativePath);
+                        }
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        continue;
+                    }
+                    const extraction = await (0, index_js_1.parseFile)(content, file.absolutePath, language);
+                    if (!extraction) {
+                        parseErrors++;
+                        if (options.verbose) {
+                            process.stdout.write(`  ⚠  ${file.relativePath} (parse error)\n`);
+                        }
+                        else if (!quiet) {
+                            writeProgress(processed, total, file.relativePath);
+                        }
+                        if (processed % BATCH_SIZE === 0)
+                            closeBatch();
+                        continue;
+                    }
+                    const symbolIdMap = new Map(); // qualifiedName → id
+                    for (const def of extraction.definitions) {
+                        const symId = this.store.insertSymbol(fileId, def);
+                        const qname = def.qualifiedName ?? def.name;
+                        if (!symbolIdMap.has(qname))
+                            symbolIdMap.set(qname, symId);
+                    }
+                    for (const ref of extraction.references) {
+                        const fromId = ref.callerName ? symbolIdMap.get(ref.callerName) : undefined;
+                        if (fromId !== undefined) {
+                            this.store.insertEdge(fromId, ref.calleeName, ref.kind, ref.line);
+                        }
+                    }
+                    for (const mod of extraction.importedModules) {
+                        this.store.insertFileImport(fileId, mod);
+                    }
+                    if (extraction.routes) {
+                        for (const r of extraction.routes) {
+                            this.store.insertRoute(fileId, r.method, r.path, r.framework, r.handlerName ?? null, r.line, {
+                                protocol: r.protocol ?? 'http',
+                                operation: r.operation ?? null,
+                                topic: r.topic ?? null,
+                                queue: r.queue ?? null,
+                                exchange: r.exchange ?? null,
+                                service: r.service ?? null,
+                                broker: r.broker ?? null,
+                                metadataJson: r.metadataJson ?? null,
+                            });
+                        }
+                    }
+                    if (extraction.configKeys) {
+                        for (const c of extraction.configKeys) {
+                            const enclosingId = c.callerName ? symbolIdMap.get(c.callerName) ?? null : null;
+                            this.store.insertConfigKey(c.key, c.source, fileId, enclosingId, c.line);
+                        }
+                    }
+                    if (extraction.serviceCalls) {
+                        for (const sc of extraction.serviceCalls) {
+                            const enclosingId = sc.callerName ? symbolIdMap.get(sc.callerName) ?? null : null;
+                            // Only run HTTP-shaped normalization when the call is HTTP.
+                            const norm = sc.protocol === 'http'
+                                ? (0, serviceLinks_js_1.normalizeHttpTarget)(sc.rawTarget)
+                                : { path: undefined, hostHint: undefined };
+                            this.store.insertServiceCall({
+                                fileId,
+                                symbolId: enclosingId,
+                                protocol: sc.protocol,
+                                method: sc.method ?? null,
+                                rawTarget: sc.rawTarget,
+                                normalizedPath: sc.normalizedPath ?? norm.path ?? null,
+                                hostHint: sc.hostHint ?? norm.hostHint ?? null,
+                                envKey: sc.envKey ?? null,
+                                framework: sc.framework,
+                                line: sc.line,
+                                confidence: sc.confidence,
+                                operation: sc.operation ?? null,
+                                topic: sc.topic ?? null,
+                                queue: sc.queue ?? null,
+                                exchange: sc.exchange ?? null,
+                                service: sc.service ?? null,
+                                broker: sc.broker ?? null,
+                                metadataJson: sc.metadataJson ?? null,
+                            });
+                        }
+                    }
+                    if (processed % BATCH_SIZE === 0)
+                        closeBatch();
+                    indexed++;
+                    if (options.verbose) {
+                        const symCount = extraction.definitions.length;
+                        const refCount = extraction.references.length;
+                        process.stdout.write(`  ✓  ${file.relativePath} (${symCount} symbols, ${refCount} refs)\n`);
+                    }
+                    else if (!quiet) {
+                        writeProgress(processed, total, file.relativePath);
+                    }
+                }
+                finally {
+                    // Return this file's bytes to the prefetcher budget BEFORE launching
+                    // the next slot, so the launch decision sees an up-to-date balance.
+                    if (heldBytes > 0)
+                        byteSem.release(heldBytes);
+                    // Refill the sliding window now that one slot drained.
+                    launchUpTo(ioConcurrency);
+                }
+            }
+            // Close the last partial batch before kicking off post-processing
+            // (resolveImports / resolveEdges / pruneFilesNotIn all start their own
+            // transactions and would crash if one is already open).
+            closeBatch();
+        }
+        catch (err) {
+            // Don't leave a transaction dangling — post-processing's BEGIN would
+            // throw and mask the original error.
+            rollbackBatch();
+            // Drain any in-flight prefetches so their byte budget is returned and
+            // open FDs / promises don't leak as unhandled rejections.
+            while (slots.length > 0) {
+                try {
+                    const p = await slots.shift();
+                    if (p.kind === 'ok')
+                        byteSem.release(Math.max(p.size, 1));
+                }
+                catch { /* swallow */ }
+            }
+            throw err;
+        }
+        if (!options.verbose && !quiet)
+            process.stdout.write('\n');
+        return await this.finishIndex(absRoot, start, total, indexed, reusedFromCache, skipped, skippedTooLarge, parseErrors, touchedFileIds, { verbose: options.verbose, quiet: !!quiet });
+    }
+    /**
+     * Post-parse pipeline shared by the serial and parallel branches: prune
+     * stale files, resolve imports/edges/routes/config-keys, synthesize test
+     * edges, refresh external dependencies, lazily recompute PageRank, and
+     * assemble the `IndexResult`.
+     */
+    async finishIndex(absRoot, start, total, indexed, reusedFromCache, skipped, skippedTooLarge, parseErrors, touchedFileIds, opts) {
+        const { verbose, quiet } = opts;
+        // v9 Track-H: scan .proto files for gRPC service definitions BEFORE the
+        // stale-file prune and service-link resolver run. .proto files are not
+        // part of normal tree-sitter discovery, so they must be added to
+        // touchedFileIds here; otherwise cached re-indexes would prune and
+        // recreate proto rows every time.
+        try {
+            const protoScan = await (0, protoScanner_js_1.scanProtoFiles)(absRoot, this.store);
+            for (const fileId of protoScan.fileIds)
+                touchedFileIds.add(fileId);
+        }
+        catch (err) {
+            if (verbose)
+                process.stdout.write(`  ⚠  proto scanner failed: ${err}\n`);
+        }
+        // Drop files that existed in a prior run but didn't show up this time
+        // (e.g. user added a new ignore rule, or files were removed from disk).
+        // FK cascades remove their symbols, edges, and file_imports too.
+        const prunedFiles = this.store.pruneFilesNotIn(touchedFileIds);
+        if (prunedFiles > 0 && !quiet) {
+            process.stdout.write(`  Pruned ${prunedFiles.toLocaleString()} stale file(s) from prior run\n`);
+        }
+        // Post-processing passes
+        if (!quiet)
+            process.stdout.write('  Resolving imports...\n');
+        const resolvedImports = this.store.resolveImports();
+        if (!quiet)
+            process.stdout.write('  Resolving call edges...\n');
+        const resolution = this.store.resolveEdges();
+        // Track-C: link routes to handlers, config_keys to enclosing symbol,
+        // synthesize tests edges from test-file → non-test-file calls.
+        const routesResolved = this.store.resolveRouteHandlers();
+        const configKeysResolved = this.store.resolveConfigKeySymbols();
+        const testEdgesAdded = this.store.synthesizeTestEdges();
+        // v9 Track-H: scan k8s manifests + Docker Compose for service hostnames.
+        // Passed to the resolver as evidence — host_hint hits get a confidence
+        // boost and may be classified as `service_host` link matches.
+        let hostMap;
+        try {
+            hostMap = await (0, serviceHostScanner_js_1.scanServiceHosts)(absRoot);
+        }
+        catch (err) {
+            if (verbose)
+                process.stdout.write(`  ⚠  service-host scanner failed: ${err}\n`);
+        }
+        // Track-G: deterministic service-link resolution. Runs every time, since
+        // any change in service_calls OR routes can shift link membership. The
+        // resolver itself wipes service_links before rebuilding so it's
+        // idempotent.
+        let serviceLinks = 0;
+        let serviceLinksByKind = {};
+        try {
+            const sr = (0, serviceLinks_js_1.resolveServiceLinks)(this.store, { hostMap });
+            serviceLinks = sr.linksInserted;
+            serviceLinksByKind = sr.byKind;
+            this.store.markServiceCallsBackfilled();
+        }
+        catch (err) {
+            if (verbose)
+                process.stdout.write(`  ⚠  service-link resolution failed: ${err}\n`);
+        }
+        // External dependency extraction from manifests/lockfiles. This is cheap
+        // and idempotent — clear and re-insert every full pass so deletions are
+        // reflected. We pass absRoot so the extractor finds package.json /
+        // Cargo.toml / etc. at the repo root and walks down for monorepos.
+        try {
+            const { extractExternalDependencies } = await Promise.resolve().then(() => __importStar(require('./externaldeps.js')));
+            await extractExternalDependencies(absRoot, this.store);
+        }
+        catch (err) {
+            if (verbose) {
+                process.stdout.write(`  ⚠  external dep extraction failed: ${err}\n`);
+            }
+        }
+        // ── Lazy PageRank ───────────────────────────────────────────────────────────
+        // PageRank values are a pure function of the resolved edge graph. If nothing
+        // in that graph changed this run, every previously-stored rank is still
+        // correct and we can skip the O(iterations × edges) recomputation.
+        //
+        // "Nothing changed" requires ALL of the following:
+        //   - no file was newly indexed (no new symbols/edges/imports inserted)
+        //   - no stale file was pruned (would have cascaded FK deletes,
+        //     potentially NULLing inbound edge `to_id`s)
+        //   - resolveEdges() promoted zero NULL `to_id`s to a real id
+        //   - resolveImports() promoted zero NULL `resolved_file_id`s
+        //
+        // If any of those is nonzero, the symbol set OR the resolved-edge graph
+        // could have shifted, so we recompute. This is the same correctness
+        // contract that the scale-test's "top-symbol id stability" check enforces
+        // — drift there means the predicate below missed a case.
+        //
+        // Why this is safe even on first run: when the DB is fresh, `indexed > 0`
+        // (everything is new), so the predicate fires and PageRank is computed.
+        const graphChanged = indexed > 0 ||
+            prunedFiles > 0 ||
+            resolution.sameFile + resolution.imported + resolution.global > 0 ||
+            resolvedImports > 0;
+        let pagerankRecomputed = false;
+        if (graphChanged) {
+            if (!quiet)
+                process.stdout.write('  Computing PageRank...\n');
+            const symbolIds = this.store.getAllSymbolIds();
+            const edges = this.store.getAllEdges();
+            const ranks = (0, pagerank_js_1.computePageRank)(symbolIds, edges);
+            this.store.updatePageRanks(ranks);
+            pagerankRecomputed = true;
+        }
+        else if (!quiet) {
+            process.stdout.write('  Skipping PageRank (graph unchanged)\n');
+        }
+        // ── Lazy module clustering ──────────────────────────────────────────────
+        // Same skip predicate as PageRank: the cluster is a function of the file
+        // graph + symbol PageRank, both of which stay stable when nothing changed.
+        // Always build when modules table is empty so the first opt-in to v6 runs
+        // it once, even when the index itself was a no-op.
+        let modulesRecomputed = false;
+        if (graphChanged || !this.store.hasModulesData()) {
+            if (!quiet)
+                process.stdout.write('  Clustering modules...\n');
+            try {
+                (0, modules_js_1.buildModules)(this.store);
+                modulesRecomputed = true;
+            }
+            catch (err) {
+                if (verbose)
+                    process.stdout.write(`  ⚠  module clustering failed: ${err}\n`);
+            }
+        }
+        else if (!quiet) {
+            process.stdout.write('  Skipping module clustering (graph unchanged)\n');
+        }
+        // ── v10 boundary detection ──────────────────────────────────────────────
+        // Always run when graphChanged (new files / pruned files / new edges)
+        // OR when the boundaries table is empty (first opt-in after migrating
+        // an existing DB to v10).
+        let boundariesRecomputed = false;
+        try {
+            if (graphChanged || !this.store.hasBoundariesData()) {
+                if (!quiet)
+                    process.stdout.write('  Detecting boundaries...\n');
+                const r = (0, boundaries_js_1.buildBoundaries)(absRoot, this.store);
+                this.store.replaceBoundaries(r.boundaries, r.edges);
+                boundariesRecomputed = true;
+            }
+        }
+        catch (err) {
+            if (verbose)
+                process.stdout.write(`  ⚠  boundary detection failed: ${err}\n`);
+        }
+        void boundariesRecomputed;
+        // ── Lazy shape-hash pass (Track-F structural SimHash) ──────────────────
+        // Re-indexed files delete their old symbols (no shape_hash on the new
+        // rows yet) so graphChanged covers normal updates. We ALSO run when any
+        // eligible symbol is missing a hash even on a cached/no-op run — this is
+        // the case after a pre-v7 → v7 migration where every existing file is
+        // "cached" (content hash unchanged) but the new shape_hash column starts
+        // NULL on every row. Without this second predicate the backfill would
+        // never run and `seer_duplicates` would silently return nothing.
+        let shapeHashesAdded = 0;
+        const needsHashBackfill = this.store.hasMissingShapeHashes();
+        if (graphChanged || needsHashBackfill) {
+            if (!quiet) {
+                process.stdout.write(graphChanged
+                    ? '  Computing shape hashes...\n'
+                    : '  Backfilling shape hashes...\n');
+            }
+            try {
+                const r = (0, shapehash_js_1.buildShapeHashes)(this.store);
+                shapeHashesAdded = r.symbolsHashed;
+            }
+            catch (err) {
+                if (verbose)
+                    process.stdout.write(`  ⚠  shape-hash pass failed: ${err}\n`);
+            }
+        }
+        else if (!quiet) {
+            process.stdout.write('  Skipping shape hashes (graph unchanged, no backfill needed)\n');
+        }
+        // v10 — rename/move continuity heuristics. Runs whenever shape hashes
+        // were computed; opt-in mode (includeAllSymbols=true) is reserved for
+        // the explicit `seer continuity` CLI. The default pass only attaches
+        // candidates to symbols whose recorded history is shallow (< 1 commit).
+        if (graphChanged && this.store.hasV10()) {
+            try {
+                (0, continuity_js_1.buildContinuity)(this.store, { includeAllSymbols: true });
+            }
+            catch (err) {
+                if (verbose)
+                    process.stdout.write(`  ⚠  continuity pass failed: ${err}\n`);
+            }
+        }
+        const stats = this.store.getStats();
+        const elapsedMs = Date.now() - start;
+        return {
+            filesDiscovered: total,
+            filesIndexed: indexed,
+            filesReusedFromCache: reusedFromCache,
+            filesSkipped: skipped,
+            filesSkippedTooLarge: skippedTooLarge,
+            filesParseError: parseErrors,
+            wasmResets: (0, index_js_1.wasmResetCount)() + (opts.workerWasmResets ?? 0),
+            symbols: stats.symbols,
+            edges: stats.edges,
+            // stats.resolvedEdges is the running DB total; resolution.{sameFile,
+            // imported, global} below reports only the *delta* — what this run
+            // newly resolved (mostly nonzero on first run, near-zero on a cached
+            // re-run where everything was already resolved).
+            resolvedEdges: stats.resolvedEdges,
+            resolvedImports,
+            edgeResolution: {
+                sameFile: resolution.sameFile,
+                imported: resolution.imported,
+                global: resolution.global,
+            },
+            pagerankRecomputed,
+            routesResolved,
+            configKeysResolved,
+            testEdgesAdded,
+            externalDependencies: stats.externalDependencies,
+            modules: stats.modules,
+            modulesRecomputed,
+            shapeHashesAdded,
+            serviceLinks,
+            serviceLinksByKind,
+            elapsedMs,
+        };
+    }
+}
+exports.Indexer = Indexer;
+// ── Utilities ──────────────────────────────────────────────────────────────────
+function sha256(content) {
+    return crypto_1.default.createHash('sha256').update(content, 'utf8').digest('hex').slice(0, 16);
+}
+function writeProgress(current, total, label) {
+    if (!process.stdout.isTTY)
+        return;
+    const width = 28;
+    const pct = total > 0 ? current / total : 0;
+    const filled = Math.round(pct * width);
+    const bar = '█'.repeat(filled) + '░'.repeat(width - filled);
+    const pctStr = Math.round(pct * 100).toString().padStart(3);
+    const short = label.length > 35 ? '…' + label.slice(-34) : label.padEnd(35);
+    process.stdout.write(`\r  [${bar}] ${pctStr}% (${current}/${total}) ${short}`);
+}
+//# sourceMappingURL=index.js.map