raggrep 0.10.5 → 0.10.8

package/dist/app/indexer/index.d.ts

@@ -38,14 +38,23 @@ export interface TimingInfo {
     indexingMs: number;
     /** Time spent on cleanup operations */
     cleanupMs: number;
-    /** Number of files discovered */
+    /** Number of files discovered on disk */
     filesDiscovered: number;
-    /** Number of files that needed stat check */
+    /** Number of files that had stat checks performed (may be > filesDiscovered due to multiple modules) */
     filesStatChecked: number;
-    /** Number of files that needed indexing */
-    filesIndexed: number;
+    /** Number of files with detected changes (mtime or size changed) that went to Phase 2 */
+    filesWithChanges: number;
+    /** Number of files that were actually re-indexed (content changed) */
+    filesReindexed: number;
     /** Whether result was from cache */
     fromCache: boolean;
+    /** Diagnostic: breakdown of why files went to Phase 2 */
+    phase2Reasons?: {
+        newFiles: number;
+        noFileSize: number;
+        sizeMismatch: number;
+        noContentHash: number;
+    };
 }
 export interface EnsureFreshResult {
     /** Number of files indexed (new or modified) */

package/dist/cli/main.js

@@ -12057,7 +12057,12 @@ async function ensureIndexFresh(rootDir, options = {}) {
   let cleanupMs = 0;
   let filesDiscovered = 0;
   let filesStatChecked = 0;
-  let filesIndexed = 0;
+  let filesWithChanges = 0;
+  let filesReindexed = 0;
+  let totalDiagNewFiles = 0;
+  let totalDiagNoFileSize = 0;
+  let totalDiagSizeMismatch = 0;
+  let totalDiagNoContentHash = 0;
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path22.resolve(rootDir);
   const status = await getIndexStatus(rootDir);
@@ -12099,7 +12104,8 @@ async function ensureIndexFresh(rootDir, options = {}) {
         cleanupMs: 0,
         filesDiscovered: 0,
         filesStatChecked: 0,
-        filesIndexed: 0,
+        filesWithChanges: 0,
+        filesReindexed: 0,
         fromCache: true
       };
     }
@@ -12194,41 +12200,91 @@ async function ensureIndexFresh(rootDir, options = {}) {
       try {
         const stats = await fs8.stat(filepath);
         const lastModified = stats.mtime.toISOString();
+        const fileSize = stats.size;
         const existingEntry = manifest.files[relativePath];
         if (!existingEntry) {
-          return { filepath, relativePath, lastModified, needsCheck: true, isNew: true };
+          return { filepath, relativePath, lastModified, fileSize, needsCheck: true, isNew: true };
         }
         if (existingEntry.lastModified === lastModified) {
-          return { filepath, relativePath, lastModified, needsCheck: false, isNew: false };
+          return { filepath, relativePath, lastModified, fileSize, needsCheck: false, isNew: false };
         }
-        return { filepath, relativePath, lastModified, needsCheck: true, isNew: false };
+        if (existingEntry.fileSize !== undefined && existingEntry.fileSize === fileSize && existingEntry.contentHash) {
+          return { filepath, relativePath, lastModified, fileSize, needsCheck: false, isNew: false, existingContentHash: existingEntry.contentHash };
+        }
+        return { filepath, relativePath, lastModified, fileSize, needsCheck: true, isNew: false, existingContentHash: existingEntry.contentHash };
       } catch {
         return null;
       }
     };
+    const moduleFiles = module2.supportsFile ? currentFiles.filter((f) => module2.supportsFile(f)) : currentFiles;
     const statCheckStart = Date.now();
-    const statResults = await parallelMap(currentFiles, statCheck, STAT_CONCURRENCY);
+    const statResults = await parallelMap(moduleFiles, statCheck, STAT_CONCURRENCY);
     statCheckMs += Date.now() - statCheckStart;
-    filesStatChecked += currentFiles.length;
+    filesStatChecked += moduleFiles.length;
     const filesToProcess = [];
+    const filesWithMtimeOnlyChange = [];
     let unchangedCount = 0;
+    let diagNewFiles = 0;
+    let diagNoFileSize = 0;
+    let diagSizeMismatch = 0;
+    let diagNoContentHash = 0;
     for (const result2 of statResults) {
       if (!result2.success || !result2.value)
         continue;
       if (result2.value.needsCheck) {
         filesToProcess.push(result2.value);
+        if (result2.value.isNew) {
+          diagNewFiles++;
+        } else {
+          const existingEntry = manifest.files[result2.value.relativePath];
+          if (existingEntry) {
+            if (existingEntry.fileSize === undefined)
+              diagNoFileSize++;
+            else if (existingEntry.fileSize !== result2.value.fileSize)
+              diagSizeMismatch++;
+            else if (!existingEntry.contentHash)
+              diagNoContentHash++;
+          }
+        }
       } else {
         unchangedCount++;
+        const existingEntry = manifest.files[result2.value.relativePath];
+        if (existingEntry && existingEntry.lastModified !== result2.value.lastModified) {
+          filesWithMtimeOnlyChange.push(result2.value);
+        }
+      }
+    }
+    totalDiagNewFiles += diagNewFiles;
+    totalDiagNoFileSize += diagNoFileSize;
+    totalDiagSizeMismatch += diagSizeMismatch;
+    totalDiagNoContentHash += diagNoContentHash;
+    if (filesToProcess.length > 100 && verbose) {
+      logger.info(`  [Diagnostic] Phase 2 reasons: new=${diagNewFiles}, noSize=${diagNoFileSize}, sizeMismatch=${diagSizeMismatch}, noHash=${diagNoContentHash}`);
+    }
+    let mtimeOnlyUpdates = 0;
+    for (const file of filesWithMtimeOnlyChange) {
+      const existingEntry = manifest.files[file.relativePath];
+      if (existingEntry) {
+        manifest.files[file.relativePath] = {
+          ...existingEntry,
+          lastModified: file.lastModified,
+          fileSize: file.fileSize
+        };
+        mtimeOnlyUpdates++;
       }
     }
     if (filesToProcess.length === 0) {
       totalUnchanged += unchangedCount;
+      if (mtimeOnlyUpdates > 0) {
+        manifest.lastUpdated = new Date().toISOString();
+        await writeModuleManifest(rootDir, module2.id, manifest, config);
+      }
       continue;
     }
     let completedCount = 0;
     const totalToProcess = filesToProcess.length;
     const processChangedFile = async (statResult) => {
-      const { filepath, relativePath, lastModified, isNew } = statResult;
+      const { filepath, relativePath, lastModified, fileSize, isNew } = statResult;
       try {
         const content = await fs8.readFile(filepath, "utf-8");
         const contentHash = computeContentHash(content);
@@ -12239,6 +12295,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
             relativePath,
             status: "mtime_updated",
             lastModified,
+            fileSize,
             contentHash
           };
         }
@@ -12247,13 +12304,14 @@ async function ensureIndexFresh(rootDir, options = {}) {
         introspection.addFile(relativePath, content);
         const fileIndex = await module2.indexFile(relativePath, content, ctx);
         if (!fileIndex) {
-          return { relativePath, status: "unchanged" };
+          return { relativePath, status: "unchanged", fileSize };
         }
         await writeFileIndex(rootDir, module2.id, relativePath, fileIndex, config);
         return {
           relativePath,
           status: "indexed",
           lastModified,
+          fileSize,
           chunkCount: fileIndex.chunks.length,
           contentHash
         };
@@ -12266,7 +12324,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
     const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
     const results = await parallelMap(filesToProcess, processChangedFile, concurrency);
     indexingMs += Date.now() - indexingStart;
-    filesIndexed += filesToProcess.length;
+    filesWithChanges += filesToProcess.length;
     totalUnchanged += unchangedCount;
     logger.clearProgress();
     let mtimeUpdates = 0;
@@ -12280,7 +12338,8 @@ async function ensureIndexFresh(rootDir, options = {}) {
           manifest.files[fileResult.relativePath] = {
             lastModified: fileResult.lastModified,
             chunkCount: fileResult.chunkCount,
-            contentHash: fileResult.contentHash
+            contentHash: fileResult.contentHash,
+            fileSize: fileResult.fileSize
           };
           totalIndexed++;
           break;
@@ -12289,7 +12348,8 @@ async function ensureIndexFresh(rootDir, options = {}) {
             manifest.files[fileResult.relativePath] = {
               ...manifest.files[fileResult.relativePath],
               lastModified: fileResult.lastModified,
-              contentHash: fileResult.contentHash
+              contentHash: fileResult.contentHash,
+              fileSize: fileResult.fileSize
             };
             mtimeUpdates++;
           }
@@ -12303,7 +12363,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
           break;
       }
     }
-    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0;
+    const hasManifestChanges = totalIndexed > 0 || totalRemoved > 0 || mtimeUpdates > 0 || mtimeOnlyUpdates > 0;
     if (hasManifestChanges) {
       manifest.lastUpdated = new Date().toISOString();
       await writeModuleManifest(rootDir, module2.id, manifest, config);
@@ -12337,8 +12397,15 @@ async function ensureIndexFresh(rootDir, options = {}) {
       cleanupMs,
       filesDiscovered,
       filesStatChecked,
-      filesIndexed,
-      fromCache: false
+      filesWithChanges,
+      filesReindexed: totalIndexed,
+      fromCache: false,
+      phase2Reasons: {
+        newFiles: totalDiagNewFiles,
+        noFileSize: totalDiagNoFileSize,
+        sizeMismatch: totalDiagSizeMismatch,
+        noContentHash: totalDiagNoContentHash
+      }
     };
   }
   let finalManifestMtime = currentManifestMtime;
@@ -12407,6 +12474,7 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
     try {
       const stats = await fs8.stat(filepath);
       const lastModified = stats.mtime.toISOString();
+      const fileSize = stats.size;
       const existingEntry = manifest.files[relativePath];
       if (existingEntry && existingEntry.lastModified === lastModified) {
         completedCount++;
@@ -12422,6 +12490,7 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
           relativePath,
           status: "skipped",
           lastModified,
+          fileSize,
           contentHash
         };
       }
@@ -12431,13 +12500,14 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
       const fileIndex = await module2.indexFile(relativePath, content, ctx);
       if (!fileIndex) {
         logger.debug(`  [${completedCount}/${totalFiles}] Skipped ${relativePath} (no chunks)`);
-        return { relativePath, status: "skipped" };
+        return { relativePath, status: "skipped", fileSize };
       }
       await writeFileIndex(rootDir, module2.id, relativePath, fileIndex, config);
       return {
         relativePath,
         status: "indexed",
         lastModified,
+        fileSize,
         chunkCount: fileIndex.chunks.length,
         contentHash
       };
@@ -12460,7 +12530,8 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
         manifest.files[fileResult.relativePath] = {
           lastModified: fileResult.lastModified,
           chunkCount: fileResult.chunkCount,
-          contentHash: fileResult.contentHash
+          contentHash: fileResult.contentHash,
+          fileSize: fileResult.fileSize
         };
         result.indexed++;
         break;
@@ -12471,7 +12542,8 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
             manifest.files[fileResult.relativePath] = {
               ...existingEntry,
               lastModified: fileResult.lastModified,
-              contentHash: fileResult.contentHash
+              contentHash: fileResult.contentHash,
+              fileSize: fileResult.fileSize
             };
           }
         }
@@ -14247,7 +14319,7 @@ init_logger();
 // package.json
 var package_default = {
   name: "raggrep",
-  version: "0.10.5",
+  version: "0.10.8",
   description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   type: "module",
   main: "./dist/index.js",
@@ -14592,19 +14664,28 @@ Examples:
         }
         if (flags2.timing && freshStats.timing) {
           const t = freshStats.timing;
-          console.log("┌─ Timing ─────────────────────────────────────┐");
+          console.log("┌─ Timing ───────────────────────────────────────────────┐");
           if (t.fromCache) {
-            console.log(`│  Cache hit (TTL-based)                       │`);
-            console.log(`│  Total: ${t.totalMs.toFixed(0).padStart(6)}ms                              │`);
+            console.log(`│  Cache hit (TTL-based)                                 │`);
+            console.log(`│  Total: ${t.totalMs.toFixed(0).padStart(6)}ms                                        │`);
           } else {
-            console.log(`│  File discovery: ${t.fileDiscoveryMs.toFixed(0).padStart(6)}ms (${t.filesDiscovered} files)`.padEnd(47) + "│");
-            console.log(`│  Stat checks:    ${t.statCheckMs.toFixed(0).padStart(6)}ms (${t.filesStatChecked} files)`.padEnd(47) + "│");
-            console.log(`│  Indexing:       ${t.indexingMs.toFixed(0).padStart(6)}ms (${t.filesIndexed} files)`.padEnd(47) + "│");
-            console.log(`│  Cleanup:        ${t.cleanupMs.toFixed(0).padStart(6)}ms`.padEnd(47) + "│");
-            console.log(`│  ─────────────────────────────────────────── │`);
-            console.log(`│  Total:          ${t.totalMs.toFixed(0).padStart(6)}ms`.padEnd(47) + "│");
+            console.log(`│  File discovery: ${t.fileDiscoveryMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesDiscovered).padStart(6)} files found`.padEnd(57) + "│");
+            console.log(`│  Stat checks:    ${t.statCheckMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesStatChecked).padStart(6)} stats checked`.padEnd(57) + "│");
+            console.log(`│  Indexing:       ${t.indexingMs.toFixed(0).padStart(6)}ms  │  ${String(t.filesWithChanges).padStart(6)} with changes → ${t.filesReindexed} reindexed`.padEnd(57) + "│");
+            console.log(`│  Cleanup:        ${t.cleanupMs.toFixed(0).padStart(6)}ms  │`.padEnd(57) + "│");
+            console.log(`│  ───────────────────────────────────────────────────── │`);
+            console.log(`│  Total:          ${t.totalMs.toFixed(0).padStart(6)}ms`.padEnd(57) + "│");
+            if (t.phase2Reasons && t.filesWithChanges > 0) {
+              const r = t.phase2Reasons;
+              console.log(`│  ───────────────────────────────────────────────────── │`);
+              console.log(`│  Phase 2 breakdown (why files needed verification):    │`);
+              console.log(`│    New files:        ${String(r.newFiles).padStart(6)}`.padEnd(57) + "│");
+              console.log(`│    No cached size:   ${String(r.noFileSize).padStart(6)}`.padEnd(57) + "│");
+              console.log(`│    Size mismatch:    ${String(r.sizeMismatch).padStart(6)}`.padEnd(57) + "│");
+              console.log(`│    No cached hash:   ${String(r.noContentHash).padStart(6)}`.padEnd(57) + "│");
+            }
           }
-          console.log(`└──────────────────────────────────────────────┘
+          console.log(`└────────────────────────────────────────────────────────┘
 `);
         }
         const filePatterns = flags2.fileType ? [`*.${flags2.fileType}`] : undefined;
@@ -14864,4 +14945,4 @@ Run 'raggrep <command> --help' for more information.
 }
 main();
 
-//# debugId=D79BF0530F7DE02664756E2164756E21
+//# debugId=BAC395AB000714B764756E2164756E21