raggrep 0.8.4 → 0.9.0

package/dist/cli/main.js

@@ -614,6 +614,33 @@ class BM25Index {
   addDocument(id, tokens) {
     this.addDocuments([{ id, content: "", tokens }]);
   }
+  removeDocument(id) {
+    const doc = this.documents.get(id);
+    if (!doc)
+      return false;
+    const tokens = doc.tokens;
+    const uniqueTerms = new Set(tokens);
+    for (const term of uniqueTerms) {
+      const count = this.documentFrequencies.get(term) || 0;
+      if (count <= 1) {
+        this.documentFrequencies.delete(term);
+      } else {
+        this.documentFrequencies.set(term, count - 1);
+      }
+    }
+    const totalLength = this.avgDocLength * this.totalDocs - tokens.length;
+    this.totalDocs--;
+    this.avgDocLength = this.totalDocs > 0 ? totalLength / this.totalDocs : 0;
+    this.documents.delete(id);
+    return true;
+  }
+  updateDocument(id, newTokens) {
+    this.removeDocument(id);
+    this.addDocument(id, newTokens);
+  }
+  hasDocument(id) {
+    return this.documents.has(id);
+  }
   serialize() {
     const documents = {};
     for (const [id, { tokens }] of this.documents) {
@@ -3787,18 +3814,41 @@ class SymbolicIndex {
   addFile(summary) {
     this.fileSummaries.set(summary.filepath, summary);
   }
+  addFileIncremental(summary) {
+    const filepath = summary.filepath;
+    const oldSummary = this.fileSummaries.get(filepath);
+    this.fileSummaries.set(filepath, summary);
+    if (this.bm25Index) {
+      if (oldSummary) {
+        this.bm25Index.removeDocument(filepath);
+      }
+      const tokens = this.getTokensForSummary(filepath, summary);
+      this.bm25Index.addDocument(filepath, tokens);
+    }
+  }
   removeFile(filepath) {
     return this.fileSummaries.delete(filepath);
   }
+  removeFileIncremental(filepath) {
+    const existed = this.fileSummaries.delete(filepath);
+    if (existed && this.bm25Index) {
+      this.bm25Index.removeDocument(filepath);
+    }
+    return existed;
+  }
+  getTokensForSummary(filepath, summary) {
+    const content = [
+      ...summary.keywords,
+      ...summary.exports,
+      ...extractPathKeywords(filepath)
+    ].join(" ");
+    return tokenize(content);
+  }
   buildBM25Index() {
     this.bm25Index = new BM25Index;
     for (const [filepath, summary] of this.fileSummaries) {
-      const content = [
-        ...summary.keywords,
-        ...summary.exports,
-        ...extractPathKeywords(filepath)
-      ].join(" ");
-      this.bm25Index.addDocuments([{ id: filepath, content }]);
+      const tokens = this.getTokensForSummary(filepath, summary);
+      this.bm25Index.addDocument(filepath, tokens);
     }
     if (this.meta) {
       this.meta.fileCount = this.fileSummaries.size;
@@ -3823,6 +3873,9 @@ class SymbolicIndex {
       throw new Error("Index not initialized");
     this.meta.lastUpdated = new Date().toISOString();
     this.meta.fileCount = this.fileSummaries.size;
+    if (this.bm25Index) {
+      this.meta.bm25Serialized = this.bm25Index.serialize();
+    }
     await fs3.mkdir(this.symbolicPath, { recursive: true });
     const metaPath = path8.join(this.symbolicPath, "_meta.json");
     await fs3.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
@@ -3832,13 +3885,37 @@ class SymbolicIndex {
       await fs3.writeFile(summaryPath, JSON.stringify(summary, null, 2));
     }
   }
+  async saveIncremental(filepaths) {
+    if (!this.meta)
+      throw new Error("Index not initialized");
+    this.meta.lastUpdated = new Date().toISOString();
+    this.meta.fileCount = this.fileSummaries.size;
+    if (this.bm25Index) {
+      this.meta.bm25Serialized = this.bm25Index.serialize();
+    }
+    await fs3.mkdir(this.symbolicPath, { recursive: true });
+    const metaPath = path8.join(this.symbolicPath, "_meta.json");
+    await fs3.writeFile(metaPath, JSON.stringify(this.meta, null, 2));
+    for (const filepath of filepaths) {
+      const summary = this.fileSummaries.get(filepath);
+      if (summary) {
+        const summaryPath = this.getFileSummaryPath(filepath);
+        await fs3.mkdir(path8.dirname(summaryPath), { recursive: true });
+        await fs3.writeFile(summaryPath, JSON.stringify(summary, null, 2));
+      }
+    }
+  }
   async load() {
     const metaPath = path8.join(this.symbolicPath, "_meta.json");
     const metaContent = await fs3.readFile(metaPath, "utf-8");
     this.meta = JSON.parse(metaContent);
     this.fileSummaries.clear();
     await this.loadFileSummariesRecursive(this.symbolicPath);
-    this.buildBM25Index();
+    if (this.meta?.bm25Serialized) {
+      this.bm25Index = BM25Index.deserialize(this.meta.bm25Serialized);
+    } else {
+      this.buildBM25Index();
+    }
   }
   async loadFileSummariesRecursive(dir) {
     try {
@@ -4222,11 +4299,14 @@ class TypeScriptModule {
     const indexDir = getRaggrepDir(ctx.rootDir, ctx.config);
     this.symbolicIndex = new SymbolicIndex(indexDir, this.id);
     await this.symbolicIndex.initialize();
+    const updatedFilepaths = [];
     for (const [filepath, summary] of this.pendingSummaries) {
-      this.symbolicIndex.addFile(summary);
+      this.symbolicIndex.addFileIncremental(summary);
+      updatedFilepaths.push(filepath);
+    }
+    if (updatedFilepaths.length > 0) {
+      await this.symbolicIndex.saveIncremental(updatedFilepaths);
     }
-    this.symbolicIndex.buildBM25Index();
-    await this.symbolicIndex.save();
     this.literalIndex = new LiteralIndex(indexDir, this.id);
     await this.literalIndex.initialize();
     const indexedFilepaths = new Set;
@@ -4578,11 +4658,14 @@ class JsonModule {
     const indexDir = getRaggrepDir(ctx.rootDir, ctx.config);
     this.symbolicIndex = new SymbolicIndex(indexDir, this.id);
     await this.symbolicIndex.initialize();
+    const updatedFilepaths = [];
     for (const [filepath, summary] of this.pendingSummaries) {
-      this.symbolicIndex.addFile(summary);
+      this.symbolicIndex.addFileIncremental(summary);
+      updatedFilepaths.push(filepath);
+    }
+    if (updatedFilepaths.length > 0) {
+      await this.symbolicIndex.saveIncremental(updatedFilepaths);
     }
-    this.symbolicIndex.buildBM25Index();
-    await this.symbolicIndex.save();
     this.literalIndex = new LiteralIndex(indexDir, this.id);
     await this.literalIndex.initialize();
     const indexedFilepaths = new Set;
@@ -4909,11 +4992,14 @@ ${section.content}` : section.content,
     const indexDir = getRaggrepDir(ctx.rootDir, ctx.config);
     this.symbolicIndex = new SymbolicIndex(indexDir, this.id);
     await this.symbolicIndex.initialize();
+    const updatedFilepaths = [];
     for (const [filepath, summary] of this.pendingSummaries) {
-      this.symbolicIndex.addFile(summary);
+      this.symbolicIndex.addFileIncremental(summary);
+      updatedFilepaths.push(filepath);
+    }
+    if (updatedFilepaths.length > 0) {
+      await this.symbolicIndex.saveIncremental(updatedFilepaths);
     }
-    this.symbolicIndex.buildBM25Index();
-    await this.symbolicIndex.save();
     this.pendingSummaries.clear();
   }
   async search(query, ctx, options = {}) {
@@ -5500,6 +5586,10 @@ import { glob } from "glob";
 import * as fs7 from "fs/promises";
 import * as path16 from "path";
 import * as os3 from "os";
+import * as crypto2 from "crypto";
+function computeContentHash(content) {
+  return crypto2.createHash("sha256").update(content, "utf-8").digest("hex");
+}
 async function parallelMap(items, processor, concurrency) {
   const results = new Array(items.length);
   let nextIndex = 0;
@@ -5749,42 +5839,93 @@ async function ensureIndexFresh(rootDir, options = {}) {
       getIntrospection: (filepath) => introspection.getFile(filepath)
     };
     const totalFiles = currentFiles.length;
-    for (let i = 0;i < currentFiles.length; i++) {
-      const filepath = currentFiles[i];
+    let completedCount = 0;
+    const processIncrementalFile = async (filepath) => {
       const relativePath = path16.relative(rootDir, filepath);
-      const progress = `[${i + 1}/${totalFiles}]`;
       try {
         const stats = await fs7.stat(filepath);
         const lastModified = stats.mtime.toISOString();
         const existingEntry = manifest.files[relativePath];
         if (existingEntry && existingEntry.lastModified === lastModified) {
-          totalUnchanged++;
-          continue;
+          completedCount++;
+          return { relativePath, status: "unchanged" };
         }
-        logger.progress(`  ${progress} Indexing: ${relativePath}`);
         const content = await fs7.readFile(filepath, "utf-8");
-        introspection.addFile(relativePath, content);
-        const fileIndex = await module.indexFile(relativePath, content, ctx);
-        if (fileIndex) {
-          await writeFileIndex(rootDir, module.id, relativePath, fileIndex, config);
-          manifest.files[relativePath] = {
+        const contentHash = computeContentHash(content);
+        if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+          completedCount++;
+          return {
+            relativePath,
+            status: "mtime_updated",
             lastModified,
-            chunkCount: fileIndex.chunks.length
+            contentHash
           };
-          totalIndexed++;
         }
+        completedCount++;
+        logger.progress(`  [${completedCount}/${totalFiles}] Indexing: ${relativePath}`);
+        introspection.addFile(relativePath, content);
+        const fileIndex = await module.indexFile(relativePath, content, ctx);
+        if (!fileIndex) {
+          return { relativePath, status: "unchanged" };
+        }
+        await writeFileIndex(rootDir, module.id, relativePath, fileIndex, config);
+        return {
+          relativePath,
+          status: "indexed",
+          lastModified,
+          chunkCount: fileIndex.chunks.length,
+          contentHash
+        };
       } catch (error) {
-        logger.clearProgress();
-        logger.error(`  ${progress} Error indexing ${relativePath}: ${error}`);
+        completedCount++;
+        return { relativePath, status: "error", error };
       }
-    }
+    };
+    const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+    const results = await parallelMap(currentFiles, processIncrementalFile, concurrency);
     logger.clearProgress();
-    if (totalIndexed > 0 || totalRemoved > 0) {
+    let mtimeUpdates = 0;
+    for (const item of results) {
+      if (!item.success) {
+        continue;
+      }
+      const fileResult = item.value;
+      switch (fileResult.status) {
+        case "indexed":
+          manifest.files[fileResult.relativePath] = {
+            lastModified: fileResult.lastModified,
+            chunkCount: fileResult.chunkCount,
+            contentHash: fileResult.contentHash
+          };
+          totalIndexed++;
+          break;
+        case "mtime_updated":
+          if (manifest.files[fileResult.relativePath]) {
+            manifest.files[fileResult.relativePath] = {
+              ...manifest.files[fileResult.relativePath],
+              lastModified: fileResult.lastModified,
+              contentHash: fileResult.contentHash
+            };
+            mtimeUpdates++;
+          }
+          totalUnchanged++;
+          break;
+        case "unchanged":
+          totalUnchanged++;
+          break;
+        case "error":
+          logger.error(`  Error indexing ${fileResult.relativePath}: ${fileResult.error}`);
+          break;
+      }
+    }
+    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0;
+    if (hasManifestChanges) {
       manifest.lastUpdated = new Date().toISOString();
       await writeModuleManifest(rootDir, module.id, manifest, config);
-      if (module.finalize) {
-        await module.finalize(ctx);
-      }
+    }
+    const hasContentChanges = totalIndexed > 0 || totalRemoved > 0;
+    if (hasContentChanges && module.finalize) {
+      await module.finalize(ctx);
     }
     if (totalRemoved > 0) {
       await cleanupEmptyDirectories(indexPath);
@@ -5862,6 +6003,17 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
         return { relativePath, status: "skipped" };
       }
       const content = await fs7.readFile(filepath, "utf-8");
+      const contentHash = computeContentHash(content);
+      if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+        completedCount++;
+        logger.debug(`  [${completedCount}/${totalFiles}] Skipped ${relativePath} (content unchanged)`);
+        return {
+          relativePath,
+          status: "skipped",
+          lastModified,
+          contentHash
+        };
+      }
       introspection.addFile(relativePath, content);
       completedCount++;
       logger.progress(`  [${completedCount}/${totalFiles}] Processing: ${relativePath}`);
@@ -5875,7 +6027,8 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
         relativePath,
         status: "indexed",
         lastModified,
-        chunkCount: fileIndex.chunks.length
+        chunkCount: fileIndex.chunks.length,
+        contentHash
       };
     } catch (error) {
       completedCount++;
@@ -5895,11 +6048,22 @@ async function indexWithModule(rootDir, files, module, config, verbose, introspe
       case "indexed":
         manifest.files[fileResult.relativePath] = {
           lastModified: fileResult.lastModified,
-          chunkCount: fileResult.chunkCount
+          chunkCount: fileResult.chunkCount,
+          contentHash: fileResult.contentHash
         };
         result.indexed++;
         break;
       case "skipped":
+        if (fileResult.lastModified && fileResult.contentHash) {
+          const existingEntry = manifest.files[fileResult.relativePath];
+          if (existingEntry) {
+            manifest.files[fileResult.relativePath] = {
+              ...existingEntry,
+              lastModified: fileResult.lastModified,
+              contentHash: fileResult.contentHash
+            };
+          }
+        }
         result.skipped++;
         break;
       case "error":
@@ -7674,7 +7838,7 @@ init_logger();
 // package.json
 var package_default = {
   name: "raggrep",
-  version: "0.8.4",
+  version: "0.9.0",
   description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   type: "module",
   main: "./dist/index.js",
@@ -8267,4 +8431,4 @@ Run 'raggrep <command> --help' for more information.
 }
 main();
 
-//# debugId=96F7EF5714B8338A64756E2164756E21
+//# debugId=94148A85E332DDBC64756E2164756E21