raggrep 0.10.8 → 0.11.0

package/dist/domain/entities/fileIndex.d.ts

@@ -36,11 +36,6 @@ export interface FileManifestEntry {
      * This prevents false positives when git updates mtime on branch switches.
      */
     contentHash?: string;
-    /**
-     * File size in bytes. Used as a quick filter:
-     * If mtime changed but size is same, content is likely unchanged.
-     */
-    fileSize?: number;
 }
 /**
  * Manifest tracking all indexed files for a specific module.
@@ -61,8 +56,14 @@ export interface ModuleManifest {
 export interface GlobalManifest {
     /** RAGgrep version */
     version: string;
-    /** ISO timestamp of last update */
+    /** ISO timestamp of last update (when index files were written) */
     lastUpdated: string;
+    /**
+     * ISO timestamp of when the last index run started.
+     * Used to detect files modified since the last indexing pass.
+     * This is captured at the START of indexing, before any files are processed.
+     */
+    lastIndexStarted: string;
     /** List of active module IDs */
     modules: string[];
 }

package/dist/index.js

@@ -11973,8 +11973,8 @@ function getOptimalConcurrency() {
   return optimal;
 }
 var DEFAULT_CONCURRENCY = getOptimalConcurrency();
-var STAT_CONCURRENCY = Math.max(32, getOptimalConcurrency() * 4);
 async function indexDirectory(rootDir, options = {}) {
+  const indexStartTime = new Date().toISOString();
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
   const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
@@ -12055,7 +12055,7 @@ Indexing complete in ${formatDuration(overallDuration)}`);
   const totalSkipped = results.reduce((sum, r) => sum + r.skipped, 0);
   const totalErrors = results.reduce((sum, r) => sum + r.errors, 0);
   logger.info(`Total: ${totalIndexed} indexed, ${totalSkipped} skipped, ${totalErrors} errors`);
-  await updateGlobalManifest(rootDir, enabledModules, config);
+  await updateGlobalManifest(rootDir, enabledModules, config, indexStartTime);
   return results;
 }
 async function isIndexVersionCompatible(rootDir) {
@@ -12092,19 +12092,14 @@ async function ensureIndexFresh(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
   const showTiming = options.timing ?? false;
+  const indexStartTime = new Date().toISOString();
   const startTime = Date.now();
   let fileDiscoveryMs = 0;
-  let statCheckMs = 0;
   let indexingMs = 0;
   let cleanupMs = 0;
   let filesDiscovered = 0;
-  let filesStatChecked = 0;
-  let filesWithChanges = 0;
+  let filesChanged = 0;
   let filesReindexed = 0;
-  let totalDiagNewFiles = 0;
-  let totalDiagNoFileSize = 0;
-  let totalDiagSizeMismatch = 0;
-  let totalDiagNoContentHash = 0;
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path21.resolve(rootDir);
   const status = await getIndexStatus(rootDir);
@@ -12141,12 +12136,10 @@ async function ensureIndexFresh(rootDir, options = {}) {
       cachedResult.timing = {
         totalMs: Date.now() - startTime,
         fileDiscoveryMs: 0,
-        statCheckMs: 0,
         indexingMs: 0,
         cleanupMs: 0,
         filesDiscovered: 0,
-        filesStatChecked: 0,
-        filesWithChanges: 0,
+        filesChanged: 0,
         filesReindexed: 0,
         fromCache: true
       };
@@ -12158,15 +12151,20 @@ async function ensureIndexFresh(rootDir, options = {}) {
   if (enabledModules.length === 0) {
     return { indexed: 0, removed: 0, unchanged: 0 };
   }
+  const globalManifest = await loadGlobalManifest(rootDir, config);
+  const lastIndexStarted = globalManifest?.lastIndexStarted ? new Date(globalManifest.lastIndexStarted) : null;
   const fileDiscoveryStart = Date.now();
   const introspection = new IntrospectionIndex(rootDir);
-  const [, currentFiles] = await Promise.all([
+  const [, discoveryResult] = await Promise.all([
     introspection.initialize(),
-    findFiles(rootDir, config)
+    findFilesWithStats(rootDir, config, lastIndexStarted)
   ]);
   fileDiscoveryMs = Date.now() - fileDiscoveryStart;
+  const { allFiles: currentFiles, changedFiles, changedFileMtimes } = discoveryResult;
   filesDiscovered = currentFiles.length;
+  filesChanged = changedFiles.length;
   const currentFileSet = new Set(currentFiles.map((f) => path21.relative(rootDir, f)));
+  const changedFileSet = new Set(changedFiles);
   let totalIndexed = 0;
   let totalRemoved = 0;
   let totalUnchanged = 0;
@@ -12237,107 +12235,38 @@ async function ensureIndexFresh(rootDir, options = {}) {
       },
       getIntrospection: (filepath) => introspection.getFile(filepath)
     };
-    const statCheck = async (filepath) => {
+    const moduleChangedFiles = module2.supportsFile ? changedFiles.filter((f) => module2.supportsFile(f)) : changedFiles;
+    const filesToProcess = moduleChangedFiles.map((filepath) => {
       const relativePath = path21.relative(rootDir, filepath);
-      try {
-        const stats = await fs8.stat(filepath);
-        const lastModified = stats.mtime.toISOString();
-        const fileSize = stats.size;
-        const existingEntry = manifest.files[relativePath];
-        if (!existingEntry) {
-          return { filepath, relativePath, lastModified, fileSize, needsCheck: true, isNew: true };
-        }
-        if (existingEntry.lastModified === lastModified) {
-          return { filepath, relativePath, lastModified, fileSize, needsCheck: false, isNew: false };
-        }
-        if (existingEntry.fileSize !== undefined && existingEntry.fileSize === fileSize && existingEntry.contentHash) {
-          return { filepath, relativePath, lastModified, fileSize, needsCheck: false, isNew: false, existingContentHash: existingEntry.contentHash };
-        }
-        return { filepath, relativePath, lastModified, fileSize, needsCheck: true, isNew: false, existingContentHash: existingEntry.contentHash };
-      } catch {
-        return null;
-      }
-    };
-    const moduleFiles = module2.supportsFile ? currentFiles.filter((f) => module2.supportsFile(f)) : currentFiles;
-    const statCheckStart = Date.now();
-    const statResults = await parallelMap(moduleFiles, statCheck, STAT_CONCURRENCY);
-    statCheckMs += Date.now() - statCheckStart;
-    filesStatChecked += moduleFiles.length;
-    const filesToProcess = [];
-    const filesWithMtimeOnlyChange = [];
-    let unchangedCount = 0;
-    let diagNewFiles = 0;
-    let diagNoFileSize = 0;
-    let diagSizeMismatch = 0;
-    let diagNoContentHash = 0;
-    for (const result2 of statResults) {
-      if (!result2.success || !result2.value)
-        continue;
-      if (result2.value.needsCheck) {
-        filesToProcess.push(result2.value);
-        if (result2.value.isNew) {
-          diagNewFiles++;
-        } else {
-          const existingEntry = manifest.files[result2.value.relativePath];
-          if (existingEntry) {
-            if (existingEntry.fileSize === undefined)
-              diagNoFileSize++;
-            else if (existingEntry.fileSize !== result2.value.fileSize)
-              diagSizeMismatch++;
-            else if (!existingEntry.contentHash)
-              diagNoContentHash++;
-          }
-        }
-      } else {
-        unchangedCount++;
-        const existingEntry = manifest.files[result2.value.relativePath];
-        if (existingEntry && existingEntry.lastModified !== result2.value.lastModified) {
-          filesWithMtimeOnlyChange.push(result2.value);
-        }
-      }
-    }
-    totalDiagNewFiles += diagNewFiles;
-    totalDiagNoFileSize += diagNoFileSize;
-    totalDiagSizeMismatch += diagSizeMismatch;
-    totalDiagNoContentHash += diagNoContentHash;
-    if (filesToProcess.length > 100 && verbose) {
-      logger.info(`  [Diagnostic] Phase 2 reasons: new=${diagNewFiles}, noSize=${diagNoFileSize}, sizeMismatch=${diagSizeMismatch}, noHash=${diagNoContentHash}`);
-    }
-    let mtimeOnlyUpdates = 0;
-    for (const file of filesWithMtimeOnlyChange) {
-      const existingEntry = manifest.files[file.relativePath];
-      if (existingEntry) {
-        manifest.files[file.relativePath] = {
-          ...existingEntry,
-          lastModified: file.lastModified,
-          fileSize: file.fileSize
-        };
-        mtimeOnlyUpdates++;
-      }
-    }
+      const existingEntry = manifest.files[relativePath];
+      const lastModified = changedFileMtimes.get(filepath) || new Date().toISOString();
+      return {
+        filepath,
+        relativePath,
+        lastModified,
+        isNew: !existingEntry,
+        existingContentHash: existingEntry?.contentHash
+      };
+    });
+    const moduleAllFiles = module2.supportsFile ? currentFiles.filter((f) => module2.supportsFile(f)) : currentFiles;
+    const unchangedCount = moduleAllFiles.length - filesToProcess.length;
     if (filesToProcess.length === 0) {
       totalUnchanged += unchangedCount;
-      if (mtimeOnlyUpdates > 0) {
-        manifest.lastUpdated = new Date().toISOString();
-        await writeModuleManifest(rootDir, module2.id, manifest, config);
-      }
       continue;
     }
     let completedCount = 0;
     const totalToProcess = filesToProcess.length;
-    const processChangedFile = async (statResult) => {
-      const { filepath, relativePath, lastModified, fileSize, isNew } = statResult;
+    const processChangedFile = async (fileToProcess) => {
+      const { filepath, relativePath, lastModified, isNew, existingContentHash } = fileToProcess;
       try {
         const content = await fs8.readFile(filepath, "utf-8");
         const contentHash = computeContentHash(content);
-        const existingEntry = manifest.files[relativePath];
-        if (!isNew && existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+        if (!isNew && existingContentHash && existingContentHash === contentHash) {
           completedCount++;
           return {
             relativePath,
             status: "mtime_updated",
             lastModified,
-            fileSize,
             contentHash
           };
         }
@@ -12346,14 +12275,13 @@ async function ensureIndexFresh(rootDir, options = {}) {
         introspection.addFile(relativePath, content);
         const fileIndex = await module2.indexFile(relativePath, content, ctx);
         if (!fileIndex) {
-          return { relativePath, status: "unchanged", fileSize };
+          return { relativePath, status: "unchanged" };
         }
         await writeFileIndex(rootDir, module2.id, relativePath, fileIndex, config);
         return {
           relativePath,
           status: "indexed",
           lastModified,
-          fileSize,
           chunkCount: fileIndex.chunks.length,
           contentHash
         };
@@ -12366,7 +12294,6 @@ async function ensureIndexFresh(rootDir, options = {}) {
     const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
     const results = await parallelMap(filesToProcess, processChangedFile, concurrency);
     indexingMs += Date.now() - indexingStart;
-    filesWithChanges += filesToProcess.length;
     totalUnchanged += unchangedCount;
     logger.clearProgress();
     let mtimeUpdates = 0;
@@ -12380,18 +12307,17 @@ async function ensureIndexFresh(rootDir, options = {}) {
           manifest.files[fileResult.relativePath] = {
             lastModified: fileResult.lastModified,
             chunkCount: fileResult.chunkCount,
-            contentHash: fileResult.contentHash,
-            fileSize: fileResult.fileSize
+            contentHash: fileResult.contentHash
           };
           totalIndexed++;
+          filesReindexed++;
           break;
         case "mtime_updated":
           if (manifest.files[fileResult.relativePath]) {
             manifest.files[fileResult.relativePath] = {
               ...manifest.files[fileResult.relativePath],
               lastModified: fileResult.lastModified,
-              contentHash: fileResult.contentHash,
-              fileSize: fileResult.fileSize
+              contentHash: fileResult.contentHash
             };
             mtimeUpdates++;
           }
@@ -12405,7 +12331,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
           break;
       }
     }
-    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0 || mtimeOnlyUpdates > 0;
+    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0;
     if (hasManifestChanges) {
       manifest.lastUpdated = new Date().toISOString();
       await writeModuleManifest(rootDir, module2.id, manifest, config);
@@ -12421,8 +12347,8 @@ async function ensureIndexFresh(rootDir, options = {}) {
   if (totalIndexed > 0) {
     await introspection.save(config);
   }
+  await updateGlobalManifest(rootDir, enabledModules, config, indexStartTime);
   if (totalIndexed > 0 || totalRemoved > 0) {
-    await updateGlobalManifest(rootDir, enabledModules, config);
     clearFreshnessCache();
   }
   const result = {
@@ -12434,20 +12360,12 @@ async function ensureIndexFresh(rootDir, options = {}) {
     result.timing = {
       totalMs: Date.now() - startTime,
       fileDiscoveryMs,
-      statCheckMs,
       indexingMs,
       cleanupMs,
       filesDiscovered,
-      filesStatChecked,
-      filesWithChanges,
-      filesReindexed: totalIndexed,
-      fromCache: false,
-      phase2Reasons: {
-        newFiles: totalDiagNewFiles,
-        noFileSize: totalDiagNoFileSize,
-        sizeMismatch: totalDiagSizeMismatch,
-        noContentHash: totalDiagNoContentHash
-      }
+      filesChanged,
+      filesReindexed,
+      fromCache: false
     };
   }
   let finalManifestMtime = currentManifestMtime;
@@ -12516,7 +12434,6 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
     try {
       const stats = await fs8.stat(filepath);
       const lastModified = stats.mtime.toISOString();
-      const fileSize = stats.size;
       const existingEntry = manifest.files[relativePath];
       if (existingEntry && existingEntry.lastModified === lastModified) {
         completedCount++;
@@ -12532,7 +12449,6 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
           relativePath,
           status: "skipped",
           lastModified,
-          fileSize,
           contentHash
         };
       }
@@ -12542,14 +12458,13 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
       const fileIndex = await module2.indexFile(relativePath, content, ctx);
       if (!fileIndex) {
         logger.debug(`  [${completedCount}/${totalFiles}] Skipped ${relativePath} (no chunks)`);
-        return { relativePath, status: "skipped", fileSize };
+        return { relativePath, status: "skipped" };
       }
       await writeFileIndex(rootDir, module2.id, relativePath, fileIndex, config);
       return {
         relativePath,
         status: "indexed",
         lastModified,
-        fileSize,
         chunkCount: fileIndex.chunks.length,
         contentHash
       };
@@ -12572,8 +12487,7 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
         manifest.files[fileResult.relativePath] = {
           lastModified: fileResult.lastModified,
           chunkCount: fileResult.chunkCount,
-          contentHash: fileResult.contentHash,
-          fileSize: fileResult.fileSize
+          contentHash: fileResult.contentHash
         };
         result.indexed++;
         break;
@@ -12584,8 +12498,7 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
             manifest.files[fileResult.relativePath] = {
               ...existingEntry,
               lastModified: fileResult.lastModified,
-              contentHash: fileResult.contentHash,
-              fileSize: fileResult.fileSize
+              contentHash: fileResult.contentHash
             };
           }
         }
@@ -12601,15 +12514,50 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
   await writeModuleManifest(rootDir, module2.id, manifest, config);
   return result;
 }
-async function findFiles(rootDir, config) {
+var STAT_CONCURRENCY = 64;
+async function findFilesWithStats(rootDir, config, lastIndexStarted) {
   const validExtensions = new Set(config.extensions);
   const ignoreDirs = new Set(config.ignorePaths);
+  const lastIndexMs = lastIndexStarted?.getTime() ?? 0;
   const crawler = new Builder().withFullPaths().exclude((dirName) => ignoreDirs.has(dirName)).filter((filePath) => {
     const ext = path21.extname(filePath);
     return validExtensions.has(ext);
   }).crawl(rootDir);
-  const files = await crawler.withPromise();
-  return files;
+  const allFiles = await crawler.withPromise();
+  if (!lastIndexStarted) {
+    const changedFileMtimes2 = new Map;
+    await parallelMap(allFiles, async (filePath) => {
+      try {
+        const stats = await fs8.stat(filePath);
+        changedFileMtimes2.set(filePath, stats.mtime.toISOString());
+      } catch {}
+    }, STAT_CONCURRENCY);
+    return {
+      allFiles,
+      changedFiles: allFiles,
+      changedFileMtimes: changedFileMtimes2
+    };
+  }
+  const changedFiles = [];
+  const changedFileMtimes = new Map;
+  await parallelMap(allFiles, async (filePath) => {
+    try {
+      const stats = await fs8.stat(filePath);
+      if (stats.mtimeMs > lastIndexMs) {
+        changedFiles.push(filePath);
+        changedFileMtimes.set(filePath, stats.mtime.toISOString());
+      }
+    } catch {}
+  }, STAT_CONCURRENCY);
+  return {
+    allFiles,
+    changedFiles,
+    changedFileMtimes
+  };
+}
+async function findFiles(rootDir, config) {
+  const result = await findFilesWithStats(rootDir, config, null);
+  return result.allFiles;
 }
 async function loadModuleManifest(rootDir, moduleId, config) {
   const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
@@ -12636,11 +12584,21 @@ async function writeFileIndex(rootDir, moduleId, filepath, fileIndex, config) {
   await fs8.mkdir(path21.dirname(indexFilePath), { recursive: true });
   await fs8.writeFile(indexFilePath, JSON.stringify(fileIndex, null, 2));
 }
-async function updateGlobalManifest(rootDir, modules, config) {
+async function loadGlobalManifest(rootDir, config) {
+  const manifestPath = getGlobalManifestPath(rootDir, config);
+  try {
+    const content = await fs8.readFile(manifestPath, "utf-8");
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+async function updateGlobalManifest(rootDir, modules, config, indexStartTime) {
   const manifestPath = getGlobalManifestPath(rootDir, config);
   const manifest = {
     version: INDEX_SCHEMA_VERSION,
     lastUpdated: new Date().toISOString(),
+    lastIndexStarted: indexStartTime,
     modules: modules.map((m) => m.id)
   };
   await fs8.mkdir(path21.dirname(manifestPath), { recursive: true });
@@ -14010,7 +13968,7 @@ async function search(rootDir, query, options = {}) {
   console.log(`Searching for: "${query}"`);
   const config = await loadConfig(rootDir);
   await registerBuiltInModules();
-  const globalManifest = await loadGlobalManifest(rootDir, config);
+  const globalManifest = await loadGlobalManifest2(rootDir, config);
   if (!globalManifest || globalManifest.modules.length === 0) {
     console.log('No index found. Run "raggrep index" first.');
     return [];
@@ -14094,7 +14052,7 @@ async function traverseDirectory(dir, files, basePath) {
     }
   } catch {}
 }
-async function loadGlobalManifest(rootDir, config) {
+async function loadGlobalManifest2(rootDir, config) {
   const manifestPath = getGlobalManifestPath(rootDir, config);
   try {
     const content = await fs9.readFile(manifestPath, "utf-8");
@@ -14187,4 +14145,4 @@ export {
   ConsoleLogger
 };
 
-//# debugId=3F7DE957CF0C467164756E2164756E21
+//# debugId=EA3CF8830C1070D164756E2164756E21