viberag 0.7.0 → 0.8.0

package/dist/daemon/services/v2/indexing.js

@@ -28,6 +28,7 @@ import { extractV2FromFile, } from './extract/extract.js';
 import { StorageV2 } from './storage/index.js';
 import { checkV2IndexCompatibility, loadV2Manifest, saveV2Manifest, V2ReindexRequiredError, V2_SCHEMA_VERSION, } from './manifest.js';
 import { TypedEmitter, } from '../types.js';
+const PERSIST_BATCH_SIZE = 500;
 let globalV2IndexPromise = null;
 export class IndexingServiceV2 extends TypedEmitter {
     constructor(projectRoot, options) {
@@ -152,6 +153,9 @@ export class IndexingServiceV2 extends TypedEmitter {
         const { force = false } = options;
         this.suppressEvents = false;
         const startTimeMs = Date.now();
+        const extracted = [];
+        const uniqueByHash = new Map();
+        const cached = new Map();
         const stats = {
             filesScanned: 0,
             filesIndexed: 0,
@@ -288,7 +292,6 @@ export class IndexingServiceV2 extends TypedEmitter {
             }
             // Extract deterministic facts
             this.emitIndexProgress('chunk', 'Extracting symbols/chunks/files', 0, filesToProcess.length, 'files');
-            const extracted = [];
             let extractedFiles = 0;
             for (const filePath of filesToProcess) {
                 throwIfAborted(this.abortSignal, 'Indexing cancelled');
@@ -312,54 +315,42 @@ export class IndexingServiceV2 extends TypedEmitter {
                 }
             }
             throwIfAborted(this.abortSignal, 'Indexing cancelled');
-            // Embed all surfaces (cached by embed_hash)
-            const embedItems = [];
+            // Dedupe embedding surfaces by hash (cache key)
+            const addEmbedInput = (hash, text, meta) => {
+                if (!uniqueByHash.has(hash)) {
+                    uniqueByHash.set(hash, { text, meta });
+                }
+            };
             for (const item of extracted) {
-                embedItems.push({
-                    hash: item.file.embed_hash,
-                    text: item.file.embed_input,
-                    meta: {
-                        filepath: item.file.file_path,
-                        startLine: 1,
-                        endLine: 1,
-                        size: item.file.embed_input.length,
-                    },
+                addEmbedInput(item.file.embed_hash, item.file.embed_input, {
+                    filepath: item.file.file_path,
+                    startLine: 1,
+                    endLine: 1,
+                    size: item.file.embed_input.length,
                 });
                 for (const s of item.symbols) {
-                    embedItems.push({
-                        hash: s.embed_hash,
-                        text: s.embed_input,
-                        meta: {
-                            filepath: s.file_path,
-                            startLine: s.start_line,
-                            endLine: s.end_line,
-                            size: s.embed_input.length,
-                        },
+                    addEmbedInput(s.embed_hash, s.embed_input, {
+                        filepath: s.file_path,
+                        startLine: s.start_line,
+                        endLine: s.end_line,
+                        size: s.embed_input.length,
                     });
                 }
                 for (const c of item.chunks) {
-                    embedItems.push({
-                        hash: c.embed_hash,
-                        text: c.embed_input,
-                        meta: {
-                            filepath: c.file_path,
-                            startLine: c.start_line,
-                            endLine: c.end_line,
-                            size: c.embed_input.length,
-                        },
+                    addEmbedInput(c.embed_hash, c.embed_input, {
+                        filepath: c.file_path,
+                        startLine: c.start_line,
+                        endLine: c.end_line,
+                        size: c.embed_input.length,
                     });
                 }
             }
-            // Dedupe by hash (cache key)
-            const uniqueByHash = new Map();
-            for (const item of embedItems) {
-                if (!uniqueByHash.has(item.hash)) {
-                    uniqueByHash.set(item.hash, { text: item.text, meta: item.meta });
-                }
-            }
             const uniqueHashes = [...uniqueByHash.keys()];
             this.emitIndexProgress('embed', 'Checking embedding cache', 0, uniqueHashes.length, 'chunks');
-            const cached = await storage.getCachedEmbeddings(uniqueHashes);
+            const cachedEmbeddings = await storage.getCachedEmbeddings(uniqueHashes);
+            for (const [hash, vector] of cachedEmbeddings) {
+                cached.set(hash, vector);
+            }
             const cacheHits = uniqueHashes.filter(h => cached.has(h));
             stats.embeddingsCached += cacheHits.length;
             const misses = uniqueHashes.filter(h => !cached.has(h));
@@ -408,6 +399,8 @@ export class IndexingServiceV2 extends TypedEmitter {
                 this.emitIndexProgress('embed', 'Embedding surfaces', embeddedSoFar, uniqueHashes.length, 'chunks');
                 await Promise.all(batchTasks.map(task => batchLimit(task)));
             }
+            // Release raw embedding inputs once vectors are available.
+            uniqueByHash.clear();
             throwIfAborted(this.abortSignal, 'Indexing cancelled');
             // Persist (upsert) rows
             this.emitIndexProgress('persist', 'Writing tables', 0, 0, null);
@@ -415,6 +408,77 @@ export class IndexingServiceV2 extends TypedEmitter {
             const symbolRows = [];
             const chunkRows = [];
             const refRows = [];
+            const clearPersistBatch = () => {
+                fileRows.length = 0;
+                symbolRows.length = 0;
+                chunkRows.length = 0;
+                refRows.length = 0;
+            };
+            const collectPersistBatchFilePaths = () => {
+                const paths = new Set();
+                const collect = (rows) => {
+                    for (const row of rows) {
+                        const filePath = row['file_path'];
+                        if (typeof filePath === 'string') {
+                            paths.add(filePath);
+                        }
+                    }
+                };
+                collect(fileRows);
+                collect(symbolRows);
+                collect(chunkRows);
+                collect(refRows);
+                return [...paths];
+            };
+            const rollbackPersistBatch = async (filePaths) => {
+                if (filePaths.length === 0) {
+                    return;
+                }
+                this.log('warn', `Persist batch failed; rolling back ${filePaths.length} file(s).`);
+                for (const filePath of filePaths) {
+                    try {
+                        await storage.deleteAllRowsForFile(filePath);
+                    }
+                    catch (error) {
+                        const message = error instanceof Error ? error.message : String(error);
+                        this.log('warn', `Rollback failed for ${filePath}: ${message}`);
+                    }
+                }
+            };
+            const flushPersistBatch = async (force = false) => {
+                // Keep persisted row buffers bounded so large repositories do not
+                // retain all rows in memory until the very end of indexing.
+                const shouldFlush = force ||
+                    fileRows.length >= PERSIST_BATCH_SIZE ||
+                    symbolRows.length >= PERSIST_BATCH_SIZE ||
+                    chunkRows.length >= PERSIST_BATCH_SIZE ||
+                    refRows.length >= PERSIST_BATCH_SIZE;
+                if (!shouldFlush)
+                    return;
+                const fileCount = fileRows.length;
+                const symbolCount = symbolRows.length;
+                const chunkCount = chunkRows.length;
+                const refCount = refRows.length;
+                const batchFilePaths = collectPersistBatchFilePaths();
+                try {
+                    await storage.upsertFiles(fileRows);
+                    await storage.upsertSymbols(symbolRows);
+                    await storage.upsertChunks(chunkRows);
+                    await storage.upsertRefs(refRows);
+                    stats.fileRowsUpserted += fileCount;
+                    stats.symbolRowsUpserted += symbolCount;
+                    stats.chunkRowsUpserted += chunkCount;
+                    stats.refRowsUpserted += refCount;
+                }
+                catch (error) {
+                    await rollbackPersistBatch(batchFilePaths);
+                    throw error;
+                }
+                finally {
+                    // Always release row buffers, even if persist fails mid-batch.
+                    clearPersistBatch();
+                }
+            };
             for (const item of extracted) {
                 const vecFile = cached.get(item.file.embed_hash);
                 if (!vecFile) {
@@ -518,15 +582,12 @@ export class IndexingServiceV2 extends TypedEmitter {
                         imported_name: r.imported_name,
                     });
                 }
+                await flushPersistBatch();
             }
-            await storage.upsertFiles(fileRows);
-            await storage.upsertSymbols(symbolRows);
-            await storage.upsertChunks(chunkRows);
-            await storage.upsertRefs(refRows);
-            stats.fileRowsUpserted += fileRows.length;
-            stats.symbolRowsUpserted += symbolRows.length;
-            stats.chunkRowsUpserted += chunkRows.length;
-            stats.refRowsUpserted += refRows.length;
+            await flushPersistBatch(true);
+            // Explicitly release large temporary structures before finalize phase.
+            cached.clear();
+            extracted.length = 0;
             const totalSymbols = await storage.getSymbolsTable().countRows();
             const totalChunks = await storage.getChunksTable().countRows();
             const totalRefs = await storage.getRefsTable().countRows();
@@ -570,6 +631,12 @@ export class IndexingServiceV2 extends TypedEmitter {
             });
             throw error;
         }
+        finally {
+            // Best-effort cleanup of large temporary structures for both success and failure paths.
+            uniqueByHash.clear();
+            cached.clear();
+            extracted.length = 0;
+        }
     }
     async initialize() {
         if (this.config)

package/dist/daemon/services/v2/search/engine.js

@@ -688,22 +688,31 @@ export class SearchEngineV2 {
             this.getChunksTable(),
             this.getSymbolsTable(),
         ]);
-        await ensureFtsIndex(chunksTable, 'string_literals', {
-            baseTokenizer: 'simple',
-            lowercase: true,
-            stem: false,
-            removeStopWords: false,
-            maxTokenLength: 256,
-            withPosition: true,
-        });
-        await ensureFtsIndex(symbolsTable, 'string_literals', {
-            baseTokenizer: 'simple',
-            lowercase: true,
-            stem: false,
-            removeStopWords: false,
-            maxTokenLength: 256,
-            withPosition: true,
-        });
+        const stringLiteralQuery = toStringLiteralExactTextQuery(query);
+        if (stringLiteralQuery) {
+            await ensureFtsIndex(chunksTable, 'string_literals', {
+                baseTokenizer: 'simple',
+                lowercase: true,
+                stem: false,
+                removeStopWords: false,
+                maxTokenLength: 256,
+                withPosition: true,
+            });
+            await ensureFtsIndex(symbolsTable, 'string_literals', {
+                baseTokenizer: 'simple',
+                lowercase: true,
+                stem: false,
+                removeStopWords: false,
+                maxTokenLength: 256,
+                withPosition: true,
+            });
+        }
+        else {
+            // LanceDB string_literals FTS can exhibit pathological native-memory
+            // growth for broad multi-token queries at high limits. Fall back to
+            // code_text channels for these queries.
+            this.log('warn', 'Exact-text query has multiple tokens; skipping string_literals channel to avoid high-memory path.');
+        }
         await ensureFtsIndex(chunksTable, 'code_text', {
             baseTokenizer: 'simple',
             lowercase: true,
@@ -722,8 +731,12 @@ export class SearchEngineV2 {
         });
         const oversample = Math.min(200, Math.max(k * 8, 40));
         const [chunkStringHits, symbolStringHits, chunkHits, symbolHits] = await Promise.all([
-            this.ftsCandidatesChunks(chunksTable, query, 'string_literals', oversample, filterClause, 'chunks.string_literals'),
-            this.ftsCandidatesSymbols(symbolsTable, query, 'string_literals', oversample, filterClause, 'symbols.string_literals'),
+            stringLiteralQuery
+                ? this.ftsCandidatesChunks(chunksTable, stringLiteralQuery, 'string_literals', oversample, filterClause, 'chunks.string_literals')
+                : Promise.resolve([]),
+            stringLiteralQuery
+                ? this.ftsCandidatesSymbols(symbolsTable, stringLiteralQuery, 'string_literals', oversample, filterClause, 'symbols.string_literals')
+                : Promise.resolve([]),
             this.ftsCandidatesChunks(chunksTable, query, 'code_text', oversample, filterClause, 'chunks.code_text'),
             this.ftsCandidatesSymbols(symbolsTable, query, 'code_text', oversample, filterClause, 'symbols.code_text'),
         ]);
@@ -826,7 +839,21 @@ export class SearchEngineV2 {
         return reranked.slice(0, k).map(c => toHit(c, explain));
     }
     async ftsCandidatesSymbols(table, query, column, limit, filterClause, source) {
-        let q = table.search(query, 'fts', column).limit(limit);
+        let q = table
+            .search(query, 'fts', column)
+            .limit(limit)
+            .select([
+            'symbol_id',
+            'file_path',
+            'start_line',
+            'end_line',
+            'qualname',
+            'symbol_name',
+            'signature',
+            'code_text',
+            'is_exported',
+            '_score',
+        ]);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -854,7 +881,21 @@ export class SearchEngineV2 {
         });
     }
     async ftsCandidatesSymbolsFullTextQuery(table, query, limit, filterClause, source) {
-        let q = table.search(query, 'fts').limit(limit);
+        let q = table
+            .search(query, 'fts')
+            .limit(limit)
+            .select([
+            'symbol_id',
+            'file_path',
+            'start_line',
+            'end_line',
+            'qualname',
+            'symbol_name',
+            'signature',
+            'code_text',
+            'is_exported',
+            '_score',
+        ]);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -882,7 +923,18 @@ export class SearchEngineV2 {
         });
     }
     async ftsCandidatesChunks(table, query, column, limit, filterClause, source) {
-        let q = table.search(query, 'fts', column).limit(limit);
+        let q = table
+            .search(query, 'fts', column)
+            .limit(limit)
+            .select([
+            'chunk_id',
+            'file_path',
+            'start_line',
+            'end_line',
+            'chunk_kind',
+            'code_text',
+            '_score',
+        ]);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -908,7 +960,10 @@ export class SearchEngineV2 {
         });
     }
     async ftsCandidatesFiles(table, query, column, limit, filterClause, source) {
-        let q = table.search(query, 'fts', column).limit(limit);
+        let q = table
+            .search(query, 'fts', column)
+            .limit(limit)
+            .select(['file_id', 'file_path', 'file_summary_text', '_score']);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -935,7 +990,23 @@ export class SearchEngineV2 {
         });
     }
     async ftsCandidatesRefs(table, query, column, limit, filterClause, source) {
-        let q = table.search(query, 'fts', column).limit(limit);
+        let q = table
+            .search(query, 'fts', column)
+            .limit(limit)
+            .select([
+            'ref_id',
+            'file_path',
+            'start_line',
+            'end_line',
+            'start_byte',
+            'end_byte',
+            'ref_kind',
+            'token_texts',
+            'context_snippet',
+            'module_name',
+            'imported_name',
+            '_score',
+        ]);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -977,7 +1048,22 @@ export class SearchEngineV2 {
         });
     }
     async vectorCandidatesSymbols(table, queryVector, limit, filterClause, source) {
-        let q = table.vectorSearch(queryVector).column('vec_summary').limit(limit);
+        let q = table
+            .vectorSearch(queryVector)
+            .column('vec_summary')
+            .limit(limit)
+            .select([
+            'symbol_id',
+            'file_path',
+            'start_line',
+            'end_line',
+            'qualname',
+            'symbol_name',
+            'signature',
+            'code_text',
+            'is_exported',
+            '_distance',
+        ]);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -1000,7 +1086,19 @@ export class SearchEngineV2 {
         });
     }
     async vectorCandidatesChunks(table, queryVector, limit, filterClause, source) {
-        let q = table.vectorSearch(queryVector).column('vec_code').limit(limit);
+        let q = table
+            .vectorSearch(queryVector)
+            .column('vec_code')
+            .limit(limit)
+            .select([
+            'chunk_id',
+            'file_path',
+            'start_line',
+            'end_line',
+            'chunk_kind',
+            'code_text',
+            '_distance',
+        ]);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -1021,7 +1119,11 @@ export class SearchEngineV2 {
         });
     }
     async vectorCandidatesFiles(table, queryVector, limit, filterClause, source) {
-        let q = table.vectorSearch(queryVector).column('vec_file').limit(limit);
+        let q = table
+            .vectorSearch(queryVector)
+            .column('vec_file')
+            .limit(limit)
+            .select(['file_id', 'file_path', 'file_summary_text', '_distance']);
         if (filterClause)
             q = q.where(filterClause);
         const rows = await q.toArray();
@@ -1240,6 +1342,16 @@ function routeIntent(query) {
     }
     return 'concept';
 }
+function toStringLiteralExactTextQuery(query) {
+    const trimmed = query.trim();
+    if (!trimmed)
+        return null;
+    // Multi-token exact-text queries at high limits can trigger pathological
+    // native-memory growth in LanceDB string_literals FTS. Use string literal
+    // channels only for focused single-token probes.
+    const tokens = trimmed.split(/\s+/g).filter(Boolean);
+    return tokens.length === 1 ? trimmed : null;
+}
 function buildDefinitionFuzzyPlan(query) {
     const normalized = normalizeDefinitionFuzzyQuery(query);
     if (!normalized)

package/dist/daemon/services/v2/storage/index.d.ts

@@ -27,6 +27,7 @@ export declare class StorageV2 {
     private openOrCreateVectorTable;
     private getTableVectorDimensions;
     close(): void;
+    private safeClose;
     private getDb;
     getSymbolsTable(): Table;
     getChunksTable(): Table;

package/dist/daemon/services/v2/storage/index.js

@@ -114,6 +114,14 @@ export class StorageV2 {
         }
     }
     close() {
+        // LanceDB tables/connection own native resources; close explicitly instead
+        // of relying on GC finalizers during long-lived daemon lifecycles.
+        this.safeClose(this.cacheTable);
+        this.safeClose(this.refsTable);
+        this.safeClose(this.filesTable);
+        this.safeClose(this.chunksTable);
+        this.safeClose(this.symbolsTable);
+        this.safeClose(this.db);
         this.db = null;
         this.symbolsTable = null;
         this.chunksTable = null;
@@ -121,6 +129,16 @@ export class StorageV2 {
         this.refsTable = null;
         this.cacheTable = null;
     }
+    safeClose(target) {
+        if (!target)
+            return;
+        try {
+            target.close();
+        }
+        catch {
+            // Ignore close-time errors to avoid masking shutdown paths.
+        }
+    }
     getDb() {
         if (!this.db) {
             throw new Error('Database not connected. Call connect() first.');

package/dist/mcp/server.js

@@ -747,7 +747,7 @@ NOTE: Usually automatic via file watcher. Only call manually if needed.`,
         }),
         execute: async (args) => {
             await ensureInitialized(projectRoot);
-            const stats = await client.index({ force: args.force });
+            const stats = await client.index({ force: args.force }, 180000);
             return JSON.stringify(stats);
         },
     });
@@ -823,7 +823,7 @@ RETURNS:
 - Initialization status and setup instructions (if not initialized)
 - Index compatibility (may indicate reindex needed)
 - Index stats: file/symbol/chunk counts
-- Daemon status: running, warmup progress, indexing state
+- Daemon status: memory snapshot, warmup progress, indexing state
 
 CALL THIS FIRST if unsure whether VibeRAG is ready to use.`,
         parameters: z.object({}),
@@ -989,6 +989,7 @@ NOTE: If telemetry is disabled, this is a no-op.`,
 }
 function formatDaemonStatusSummary(status) {
     return {
+        memory: status.memory,
         warmup: {
             status: status.warmupStatus,
             elapsedMs: status.warmupElapsedMs,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "viberag",
-	"version": "0.7.0",
+	"version": "0.8.0",
 	"description": "Local code RAG for AI coding assistants - semantic search via MCP server",
 	"license": "AGPL-3.0",
 	"keywords": [
@@ -42,12 +42,19 @@
 		"dev": "tsc --watch",
 		"test": "prettier --check . && eslint . && npm run build && vitest run",
 		"test:fast": "vitest run --project=fast",
-		"test:rag": "vitest run --project=rag",
+		"test:daemon": "vitest run --project=daemon",
+		"test:rag": "npm run test:daemon",
+		"test:memory": "vitest run --project=daemon source/daemon/__tests__/*memory-regression.test.ts",
+		"test:windows-smoke": "vitest run --project=fast source/cli/__tests__/mcp-setup.test.ts source/cli/__tests__/mcp-setup-global.test.ts source/daemon/__tests__/grammar-smoke.test.ts source/daemon/__tests__/user-settings.test.ts source/mcp/__tests__/mcp-uninitialized-smoke.test.ts",
 		"test:smoke": "vitest run --testNamePattern='Grammar Smoke'",
 		"lint": "eslint .",
 		"lint:fix": "eslint . --fix",
 		"format": "prettier --write .",
 		"bake:telemetry": "node scripts/bake-telemetry-keys-local.js",
+		"memory:profile": "npm run build && node --expose-gc scripts/memory/profile.mjs",
+		"memory:isolate": "npm run build && node scripts/memory/isolate-intent-growth.mjs",
+		"memory:stress": "npm run build && node scripts/memory/stress-search-growth.mjs",
+		"memory:watch-stress": "npm run build && node scripts/memory/watch-reindex-stress.mjs",
 		"build:telemetry": "npm run build && npm run bake:telemetry",
 		"prepare": "husky",
 		"prepack": "npm run build && node scripts/bake-telemetry-keys.js",