raggrep 0.10.8 → 0.11.0

package/dist/app/indexer/index.d.ts

@@ -30,31 +30,20 @@ export interface IndexOptions {
 export interface TimingInfo {
     /** Total time in milliseconds */
     totalMs: number;
-    /** Time spent on file discovery (glob) */
+    /** Time spent on file discovery (with mtime filtering) */
     fileDiscoveryMs: number;
-    /** Time spent on stat checks */
-    statCheckMs: number;
     /** Time spent on indexing changed files */
     indexingMs: number;
     /** Time spent on cleanup operations */
     cleanupMs: number;
     /** Number of files discovered on disk */
     filesDiscovered: number;
-    /** Number of files that had stat checks performed (may be > filesDiscovered due to multiple modules) */
-    filesStatChecked: number;
-    /** Number of files with detected changes (mtime or size changed) that went to Phase 2 */
-    filesWithChanges: number;
+    /** Number of files modified since last index */
+    filesChanged: number;
     /** Number of files that were actually re-indexed (content changed) */
     filesReindexed: number;
     /** Whether result was from cache */
     fromCache: boolean;
-    /** Diagnostic: breakdown of why files went to Phase 2 */
-    phase2Reasons?: {
-        newFiles: number;
-        noFileSize: number;
-        sizeMismatch: number;
-        noContentHash: number;
-    };
 }
 export interface EnsureFreshResult {
     /** Number of files indexed (new or modified) */

package/dist/cli/main.js

@@ -11933,6 +11933,7 @@ function getOptimalConcurrency() {
   return optimal;
 }
 async function indexDirectory(rootDir, options = {}) {
+  const indexStartTime = new Date().toISOString();
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
   const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
@@ -12013,7 +12014,7 @@ Indexing complete in ${formatDuration(overallDuration)}`);
   const totalSkipped = results.reduce((sum, r) => sum + r.skipped, 0);
   const totalErrors = results.reduce((sum, r) => sum + r.errors, 0);
   logger.info(`Total: ${totalIndexed} indexed, ${totalSkipped} skipped, ${totalErrors} errors`);
-  await updateGlobalManifest(rootDir, enabledModules, config);
+  await updateGlobalManifest(rootDir, enabledModules, config, indexStartTime);
   return results;
 }
 async function isIndexVersionCompatible(rootDir) {
@@ -12050,19 +12051,14 @@ async function ensureIndexFresh(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
   const showTiming = options.timing ?? false;
+  const indexStartTime = new Date().toISOString();
   const startTime = Date.now();
   let fileDiscoveryMs = 0;
-  let statCheckMs = 0;
   let indexingMs = 0;
   let cleanupMs = 0;
   let filesDiscovered = 0;
-  let filesStatChecked = 0;
-  let filesWithChanges = 0;
+  let filesChanged = 0;
   let filesReindexed = 0;
-  let totalDiagNewFiles = 0;
-  let totalDiagNoFileSize = 0;
-  let totalDiagSizeMismatch = 0;
-  let totalDiagNoContentHash = 0;
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path22.resolve(rootDir);
   const status = await getIndexStatus(rootDir);
@@ -12099,12 +12095,10 @@ async function ensureIndexFresh(rootDir, options = {}) {
       cachedResult.timing = {
         totalMs: Date.now() - startTime,
         fileDiscoveryMs: 0,
-        statCheckMs: 0,
         indexingMs: 0,
         cleanupMs: 0,
         filesDiscovered: 0,
-        filesStatChecked: 0,
-        filesWithChanges: 0,
+        filesChanged: 0,
         filesReindexed: 0,
         fromCache: true
       };
@@ -12116,15 +12110,20 @@ async function ensureIndexFresh(rootDir, options = {}) {
   if (enabledModules.length === 0) {
     return { indexed: 0, removed: 0, unchanged: 0 };
   }
+  const globalManifest = await loadGlobalManifest(rootDir, config);
+  const lastIndexStarted = globalManifest?.lastIndexStarted ? new Date(globalManifest.lastIndexStarted) : null;
   const fileDiscoveryStart = Date.now();
   const introspection = new IntrospectionIndex(rootDir);
-  const [, currentFiles] = await Promise.all([
+  const [, discoveryResult] = await Promise.all([
     introspection.initialize(),
-    findFiles(rootDir, config)
+    findFilesWithStats(rootDir, config, lastIndexStarted)
   ]);
   fileDiscoveryMs = Date.now() - fileDiscoveryStart;
+  const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
   filesDiscovered = currentFiles.length;
+  filesChanged = changedFiles.length;
   const currentFileSet = new Set(currentFiles.map((f) => path22.relative(rootDir, f)));
+  const changedFileSet = new Set(changedFiles);
   let totalIndexed = 0;
   let totalRemoved = 0;
   let totalUnchanged = 0;
@@ -12195,107 +12194,38 @@ async function ensureIndexFresh(rootDir, options = {}) {
       },
       getIntrospection: (filepath) => introspection.getFile(filepath)
     };
-    const statCheck = async (filepath) => {
+    const moduleChangedFiles = module2.supportsFile ? changedFiles.filter((f) => module2.supportsFile(f)) : changedFiles;
+    const filesToProcess = moduleChangedFiles.map((filepath) => {
       const relativePath = path22.relative(rootDir, filepath);
-      try {
-        const stats = await fs8.stat(filepath);
-        const lastModified = stats.mtime.toISOString();
-        const fileSize = stats.size;
-        const existingEntry = manifest.files[relativePath];
-        if (!existingEntry) {
-          return { filepath, relativePath, lastModified, fileSize, needsCheck: true, isNew: true };
-        }
-        if (existingEntry.lastModified === lastModified) {
-          return { filepath, relativePath, lastModified, fileSize, needsCheck: false, isNew: false };
-        }
-        if (existingEntry.fileSize !== undefined && existingEntry.fileSize === fileSize && existingEntry.contentHash) {
-          return { filepath, relativePath, lastModified, fileSize, needsCheck: false, isNew: false, existingContentHash: existingEntry.contentHash };
-        }
-        return { filepath, relativePath, lastModified, fileSize, needsCheck: true, isNew: false, existingContentHash: existingEntry.contentHash };
-      } catch {
-        return null;
-      }
-    };
-    const moduleFiles = module2.supportsFile ? currentFiles.filter((f) => module2.supportsFile(f)) : currentFiles;
-    const statCheckStart = Date.now();
-    const statResults = await parallelMap(moduleFiles, statCheck, STAT_CONCURRENCY);
-    statCheckMs += Date.now() - statCheckStart;
-    filesStatChecked += moduleFiles.length;
-    const filesToProcess = [];
-    const filesWithMtimeOnlyChange = [];
-    let unchangedCount = 0;
-    let diagNewFiles = 0;
-    let diagNoFileSize = 0;
-    let diagSizeMismatch = 0;
-    let diagNoContentHash = 0;
-    for (const result2 of statResults) {
-      if (!result2.success || !result2.value)
-        continue;
-      if (result2.value.needsCheck) {
-        filesToProcess.push(result2.value);
-        if (result2.value.isNew) {
-          diagNewFiles++;
-        } else {
-          const existingEntry = manifest.files[result2.value.relativePath];
-          if (existingEntry) {
-            if (existingEntry.fileSize === undefined)
-              diagNoFileSize++;
-            else if (existingEntry.fileSize !== result2.value.fileSize)
-              diagSizeMismatch++;
-            else if (!existingEntry.contentHash)
-              diagNoContentHash++;
-          }
-        }
-      } else {
-        unchangedCount++;
-        const existingEntry = manifest.files[result2.value.relativePath];
-        if (existingEntry && existingEntry.lastModified !== result2.value.lastModified) {
-          filesWithMtimeOnlyChange.push(result2.value);
-        }
-      }
-    }
-    totalDiagNewFiles += diagNewFiles;
-    totalDiagNoFileSize += diagNoFileSize;
-    totalDiagSizeMismatch += diagSizeMismatch;
-    totalDiagNoContentHash += diagNoContentHash;
-    if (filesToProcess.length > 100 && verbose) {
-      logger.info(`  [Diagnostic] Phase 2 reasons: new=${diagNewFiles}, noSize=${diagNoFileSize}, sizeMismatch=${diagSizeMismatch}, noHash=${diagNoContentHash}`);
-    }
-    let mtimeOnlyUpdates = 0;
-    for (const file of filesWithMtimeOnlyChange) {
-      const existingEntry = manifest.files[file.relativePath];
-      if (existingEntry) {
-        manifest.files[file.relativePath] = {
-          ...existingEntry,
-          lastModified: file.lastModified,
-          fileSize: file.fileSize
-        };
-        mtimeOnlyUpdates++;
-      }
-    }
+      const existingEntry = manifest.files[relativePath];
+      const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
+      return {
+        filepath,
+        relativePath,
+        lastModified,
+        isNew: !existingEntry,
+        existingContentHash: existingEntry?.contentHash
+      };
+    });
+    const moduleAllFiles = module2.supportsFile ? currentFiles.filter((f) => module2.supportsFile(f)) : currentFiles;
+    const unchangedCount = moduleAllFiles.length - filesToProcess.length;
     if (filesToProcess.length === 0) {
       totalUnchanged += unchangedCount;
-      if (mtimeOnlyUpdates > 0) {
-        manifest.lastUpdated = new Date().toISOString();
-        await writeModuleManifest(rootDir, module2.id, manifest, config);
-      }
       continue;
     }
     let completedCount = 0;
     const totalToProcess = filesToProcess.length;
-    const processChangedFile = async (statResult) => {
-      const { filepath, relativePath, lastModified, fileSize, isNew } = statResult;
+    const processChangedFile = async (fileToProcess) => {
+      const { filepath, relativePath, lastModified, isNew, existingContentHash } = fileToProcess;
       try {
         const content = await fs8.readFile(filepath, "utf-8");
         const contentHash = computeContentHash(content);
-        const existingEntry = manifest.files[relativePath];
-        if (!isNew && existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+        if (!isNew && existingContentHash && existingContentHash === contentHash) {
           completedCount++;
           return {
             relativePath,
             status: "mtime_updated",
             lastModified,
-            fileSize,
             contentHash
           };
         }
@@ -12304,14 +12234,13 @@ async function ensureIndexFresh(rootDir, options = {}) {
         introspection.addFile(relativePath, content);
         const fileIndex = await module2.indexFile(relativePath, content, ctx);
         if (!fileIndex) {
-          return { relativePath, status: "unchanged", fileSize };
+          return { relativePath, status: "unchanged" };
         }
         await writeFileIndex(rootDir, module2.id, relativePath, fileIndex, config);
         return {
           relativePath,
           status: "indexed",
           lastModified,
-          fileSize,
           chunkCount: fileIndex.chunks.length,
           contentHash
         };
@@ -12324,7 +12253,6 @@ async function ensureIndexFresh(rootDir, options = {}) {
     const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
     const results = await parallelMap(filesToProcess, processChangedFile, concurrency);
     indexingMs += Date.now() - indexingStart;
-    filesWithChanges += filesToProcess.length;
     totalUnchanged += unchangedCount;
     logger.clearProgress();
     let mtimeUpdates = 0;
@@ -12338,18 +12266,17 @@ async function ensureIndexFresh(rootDir, options = {}) {
           manifest.files[fileResult.relativePath] = {
             lastModified: fileResult.lastModified,
             chunkCount: fileResult.chunkCount,
-            contentHash: fileResult.contentHash,
-            fileSize: fileResult.fileSize
+            contentHash: fileResult.contentHash
           };
           totalIndexed++;
+          filesReindexed++;
           break;
         case "mtime_updated":
           if (manifest.files[fileResult.relativePath]) {
             manifest.files[fileResult.relativePath] = {
               ...manifest.files[fileResult.relativePath],
               lastModified: fileResult.lastModified,
-              contentHash: fileResult.contentHash,
-              fileSize: fileResult.fileSize
+              contentHash: fileResult.contentHash
             };
             mtimeUpdates++;
           }
@@ -12363,7 +12290,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
           break;
       }
     }
-    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0 || mtimeOnlyUpdates > 0;
+    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0;
     if (hasManifestChanges) {
       manifest.lastUpdated = new Date().toISOString();
       await writeModuleManifest(rootDir, module2.id, manifest, config);
@@ -12379,8 +12306,8 @@ async function ensureIndexFresh(rootDir, options = {}) {
   if (totalIndexed > 0) {
     await introspection.save(config);
   }
+  await updateGlobalManifest(rootDir, enabledModules, config, indexStartTime);
   if (totalIndexed > 0 || totalRemoved > 0) {
-    await updateGlobalManifest(rootDir, enabledModules, config);
     clearFreshnessCache();
   }
   const result = {
@@ -12392,20 +12319,12 @@ async function ensureIndexFresh(rootDir, options = {}) {
     result.timing = {
       totalMs: Date.now() - startTime,
       fileDiscoveryMs,
-      statCheckMs,
       indexingMs,
       cleanupMs,
       filesDiscovered,
-      filesStatChecked,
-      filesWithChanges,
-      filesReindexed: totalIndexed,
-      fromCache: false,
-      phase2Reasons: {
-        newFiles: totalDiagNewFiles,
-        noFileSize: totalDiagNoFileSize,
-        sizeMismatch: totalDiagSizeMismatch,
-        noContentHash: totalDiagNoContentHash
-      }
+      filesChanged,
+      filesReindexed,
+      fromCache: false
     };
   }
   let finalManifestMtime = currentManifestMtime;
@@ -12474,7 +12393,6 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
     try {
       const stats = await fs8.stat(filepath);
       const lastModified = stats.mtime.toISOString();
-      const fileSize = stats.size;
       const existingEntry = manifest.files[relativePath];
       if (existingEntry && existingEntry.lastModified === lastModified) {
         completedCount++;
@@ -12490,7 +12408,6 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
           relativePath,
           status: "skipped",
           lastModified,
-          fileSize,
           contentHash
         };
       }
@@ -12500,14 +12417,13 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
       const fileIndex = await module2.indexFile(relativePath, content, ctx);
       if (!fileIndex) {
         logger.debug(`  [${completedCount}/${totalFiles}] Skipped ${relativePath} (no chunks)`);
-        return { relativePath, status: "skipped", fileSize };
+        return { relativePath, status: "skipped" };
       }
       await writeFileIndex(rootDir, module2.id, relativePath, fileIndex, config);
       return {
         relativePath,
         status: "indexed",
         lastModified,
-        fileSize,
         chunkCount: fileIndex.chunks.length,
         contentHash
       };
@@ -12530,8 +12446,7 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
         manifest.files[fileResult.relativePath] = {
           lastModified: fileResult.lastModified,
           chunkCount: fileResult.chunkCount,
-          contentHash: fileResult.contentHash,
-          fileSize: fileResult.fileSize
+          contentHash: fileResult.contentHash
         };
         result.indexed++;
         break;
@@ -12542,8 +12457,7 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
             manifest.files[fileResult.relativePath] = {
               ...existingEntry,
               lastModified: fileResult.lastModified,
-              contentHash: fileResult.contentHash,
-              fileSize: fileResult.fileSize
+              contentHash: fileResult.contentHash
             };
           }
         }
@@ -12559,15 +12473,49 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
   await writeModuleManifest(rootDir, module2.id, manifest, config);
   return result;
 }
-async function findFiles(rootDir, config) {
+async function findFilesWithStats(rootDir, config, lastIndexStarted) {
   const validExtensions = new Set(config.extensions);
   const ignoreDirs = new Set(config.ignorePaths);
+  const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
   const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
     const ext = path22.extname(filePath);
     return validExtensions.has(ext);
   }).crawl(rootDir);
-  const files = await crawler.withPromise();
-  return files;
+  const allFiles = await crawler.withPromise();
+  if (!lastIndexStarted) {
+    const changedFileMtimes2 = new Map;
+    await parallelMap(allFiles, async (filePath) => {
+      try {
+        const stats = await fs8.stat(filePath);
+        changedFileMtimes2.set(filePath, stats.mtime.toISOString());
+      } catch {}
+    }, STAT_CONCURRENCY);
+    return {
+      allFiles,
+      changedFiles: allFiles,
+      changedFileMtimes: changedFileMtimes2
+    };
+  }
+  const changedFiles = [];
+  const changedFileMtimes = new Map;
+  await parallelMap(allFiles, async (filePath) => {
+    try {
+      const stats = await fs8.stat(filePath);
+      if (stats.mtimeMs > lastIndexMs) {
+        changedFiles.push(filePath);
+        changedFileMtimes.set(filePath, stats.mtime.toISOString());
+      }
+    } catch {}
+  }, STAT_CONCURRENCY);
+  return {
+    allFiles,
+    changedFiles,
+    changedFileMtimes
+  };
+}
+async function findFiles(rootDir, config) {
+  const result = await findFilesWithStats(rootDir, config, null);
+  return result.allFiles;
 }
 async function loadModuleManifest(rootDir, moduleId, config) {
   const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
@@ -12594,11 +12542,21 @@ async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
   await fs8.mkdir(path22.dirname(indexFilePath), { recursive: true });
   await fs8.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
 }
-async function updateGlobalManifest(rootDir, modules, config) {
+async function loadGlobalManifest(rootDir, config) {
+  const manifestPath = getGlobalManifestPath(rootDir, config);
+  try {
+    const content = await fs8.readFile(manifestPath, "utf-8");
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
   const manifestPath = getGlobalManifestPath(rootDir, config);
   const manifest = {
     version: INDEX_SCHEMA_VERSION,
     lastUpdated: new Date().toISOString(),
+    lastIndexStarted: indexStartTime,
     modules: modules.map((m) => m.id)
   };
   await fs8.mkdir(path22.dirname(manifestPath), { recursive: true });
@@ -12736,7 +12694,7 @@ async function getIndexStatus(rootDir) {
   }
   return status;
 }
-var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.0.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY;
+var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.0.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY = 64;
 var init_indexer = __esm(() => {
   init_dist();
   init_config2();
@@ -12745,7 +12703,6 @@ var init_indexer = __esm(() => {
   init_logger();
   init_watcher();
   DEFAULT_CONCURRENCY = getOptimalConcurrency();
-  STAT_CONCURRENCY = Math.max(32, getOptimalConcurrency() * 4);
 });
 
 // node_modules/balanced-match/index.js
@@ -14165,7 +14122,7 @@ async function search(rootDir, query, options = {}) {
   console.log(`Searching for: "${query}"`);
   const config = await loadConfig(rootDir);
   await registerBuiltInModules();
-  const globalManifest = await loadGlobalManifest(rootDir, config);
+  const globalManifest = await loadGlobalManifest2(rootDir, config);
   if (!globalManifest || globalManifest.modules.length === 0) {
     console.log('No index found. Run "raggrep index" first.');
     return [];
@@ -14249,7 +14206,7 @@ async function traverseDirectory(dir, files, basePath) {
     }
   } catch {}
 }
-async function loadGlobalManifest(rootDir, config) {
+async function loadGlobalManifest2(rootDir, config) {
   const manifestPath = getGlobalManifestPath(rootDir, config);
   try {
     const content = await fs9.readFile(manifestPath, "utf-8");
@@ -14319,7 +14276,7 @@ init_logger();
 // package.json
 var package_default = {
   name: "raggrep",
-  version: "0.10.8",
+  version: "0.11.0",
   description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   type: "module",
   main: "./dist/index.js",
@@ -14669,21 +14626,11 @@ Examples:
             console.log(`│  Cache hit (TTL-based)                                 │`);
             console.log(`│  Total: ${t.totalMs.toFixed(0).padStart(6)}ms                                        │`);
           } else {
-            console.log(`│  File discovery: ${t.fileDiscoveryMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesDiscovered).padStart(6)} files found`.padEnd(57) + "│");
-            console.log(`│  Stat checks:    ${t.statCheckMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesStatChecked).padStart(6)} stats checked`.padEnd(57) + "│");
-            console.log(`│  Indexing:       ${t.indexingMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesWithChanges).padStart(6)} with changes → ${t.filesReindexed} reindexed`.padEnd(57) + "│");
+            console.log(`│  File discovery: ${t.fileDiscoveryMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesDiscovered).padStart(6)} files, ${t.filesChanged} changed`.padEnd(57) + "│");
+            console.log(`│  Indexing:       ${t.indexingMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesReindexed).padStart(6)} reindexed`.padEnd(57) + "│");
             console.log(`│  Cleanup:        ${t.cleanupMs.toFixed(0).padStart(6)}ms  │`.padEnd(57) + "│");
             console.log(`│  ───────────────────────────────────────────────────── │`);
             console.log(`│  Total:          ${t.totalMs.toFixed(0).padStart(6)}ms`.padEnd(57) + "│");
-            if (t.phase2Reasons && t.filesWithChanges > 0) {
-              const r = t.phase2Reasons;
-              console.log(`│  ───────────────────────────────────────────────────── │`);
-              console.log(`│  Phase 2 breakdown (why files needed verification):    │`);
-              console.log(`│    New files:        ${String(r.newFiles).padStart(6)}`.padEnd(57) + "│");
-              console.log(`│    No cached size:   ${String(r.noFileSize).padStart(6)}`.padEnd(57) + "│");
-              console.log(`│    Size mismatch:    ${String(r.sizeMismatch).padStart(6)}`.padEnd(57) + "│");
-              console.log(`│    No cached hash:   ${String(r.noContentHash).padStart(6)}`.padEnd(57) + "│");
-            }
           }
           console.log(`└────────────────────────────────────────────────────────┘
 `);
@@ -14945,4 +14892,4 @@ Run 'raggrep <command> --help' for more information.
 }
 main();
 
-//# debugId=BAC395AB000714B764756E2164756E21
+//# debugId=4E166248FB39358064756E2164756E21