raggrep 0.10.1 → 0.10.4

package/dist/index.js

@@ -11395,6 +11395,11 @@ import { watch } from "chokidar";
 init_config2();
 
 // src/app/indexer/index.ts
+var FRESHNESS_CACHE_TTL_MS = 5000;
+var freshnessCache = null;
+function clearFreshnessCache() {
+  freshnessCache = null;
+}
 function computeContentHash(content) {
   return crypto2.createHash("sha256").update(content, "utf-8").digest("hex");
 }
@@ -11436,10 +11441,12 @@ function getOptimalConcurrency() {
   return optimal;
 }
 var DEFAULT_CONCURRENCY = getOptimalConcurrency();
+var STAT_CONCURRENCY = Math.max(32, getOptimalConcurrency() * 4);
 async function indexDirectory(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
   const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+  clearFreshnessCache();
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path21.resolve(rootDir);
   const location = getIndexLocation(rootDir);
@@ -11538,6 +11545,7 @@ async function deleteIndex(rootDir) {
 }
 async function resetIndex(rootDir) {
   rootDir = path21.resolve(rootDir);
+  clearFreshnessCache();
   const status = await getIndexStatus(rootDir);
   if (!status.exists) {
     throw new Error(`No index found for ${rootDir}`);
@@ -11551,10 +11559,20 @@ async function resetIndex(rootDir) {
 async function ensureIndexFresh(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
+  const showTiming = options.timing ?? false;
+  const startTime = Date.now();
+  let fileDiscoveryMs = 0;
+  let statCheckMs = 0;
+  let indexingMs = 0;
+  let cleanupMs = 0;
+  let filesDiscovered = 0;
+  let filesStatChecked = 0;
+  let filesIndexed = 0;
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path21.resolve(rootDir);
   const status = await getIndexStatus(rootDir);
   if (!status.exists) {
+    clearFreshnessCache();
     logger.info(`No index found. Creating index...
 `);
     const results = await indexDirectory(rootDir, { ...options, logger });
@@ -11563,6 +11581,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
   }
   const versionCompatible = await isIndexVersionCompatible(rootDir);
   if (!versionCompatible) {
+    clearFreshnessCache();
     logger.info(`Index version incompatible. Rebuilding...
 `);
     await deleteIndex(rootDir);
@@ -11571,14 +11590,44 @@ async function ensureIndexFresh(rootDir, options = {}) {
     return { indexed: totalIndexed2, removed: 0, unchanged: 0 };
   }
   const config = await loadConfig(rootDir);
+  const globalManifestPath = getGlobalManifestPath(rootDir, config);
+  let currentManifestMtime = 0;
+  try {
+    const manifestStats = await fs8.stat(globalManifestPath);
+    currentManifestMtime = manifestStats.mtimeMs;
+  } catch {}
+  const now = Date.now();
+  if (freshnessCache && freshnessCache.rootDir === rootDir && now - freshnessCache.timestamp < FRESHNESS_CACHE_TTL_MS && freshnessCache.manifestMtime === currentManifestMtime) {
+    logger.debug("Using cached freshness check result");
+    const cachedResult = { ...freshnessCache.result };
+    if (showTiming) {
+      cachedResult.timing = {
+        totalMs: Date.now() - startTime,
+        fileDiscoveryMs: 0,
+        statCheckMs: 0,
+        indexingMs: 0,
+        cleanupMs: 0,
+        filesDiscovered: 0,
+        filesStatChecked: 0,
+        filesIndexed: 0,
+        fromCache: true
+      };
+    }
+    return cachedResult;
+  }
   await registerBuiltInModules();
   const enabledModules = registry.getEnabled(config);
   if (enabledModules.length === 0) {
     return { indexed: 0, removed: 0, unchanged: 0 };
   }
+  const fileDiscoveryStart = Date.now();
   const introspection = new IntrospectionIndex(rootDir);
-  await introspection.initialize();
-  const currentFiles = await findFiles(rootDir, config);
+  const [, currentFiles] = await Promise.all([
+    introspection.initialize(),
+    findFiles(rootDir, config)
+  ]);
+  fileDiscoveryMs = Date.now() - fileDiscoveryStart;
+  filesDiscovered = currentFiles.length;
   const currentFileSet = new Set(currentFiles.map((f) => path21.relative(rootDir, f)));
   let totalIndexed = 0;
   let totalRemoved = 0;
@@ -11607,20 +11656,21 @@ async function ensureIndexFresh(rootDir, options = {}) {
         filesToRemove.push(filepath);
       }
     }
+    const cleanupStart = Date.now();
     const removedFilepaths = [];
-    for (const filepath of filesToRemove) {
-      logger.debug(`  Removing stale: ${filepath}`);
-      const indexFilePath = path21.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
-      try {
-        await fs8.unlink(indexFilePath);
-      } catch {}
-      const symbolicFilePath = path21.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
-      try {
-        await fs8.unlink(symbolicFilePath);
-      } catch {}
-      delete manifest.files[filepath];
-      removedFilepaths.push(filepath);
-      totalRemoved++;
+    if (filesToRemove.length > 0) {
+      await Promise.all(filesToRemove.map(async (filepath) => {
+        logger.debug(`  Removing stale: ${filepath}`);
+        const indexFilePath = path21.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
+        const symbolicFilePath = path21.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
+        await Promise.all([
+          fs8.unlink(indexFilePath).catch(() => {}),
+          fs8.unlink(symbolicFilePath).catch(() => {})
+        ]);
+        delete manifest.files[filepath];
+        removedFilepaths.push(filepath);
+      }));
+      totalRemoved += removedFilepaths.length;
     }
     if (removedFilepaths.length > 0) {
       try {
@@ -11634,6 +11684,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
         await literalIndex.save();
       } catch {}
     }
+    cleanupMs += Date.now() - cleanupStart;
     const ctx = {
       rootDir,
       config,
@@ -11648,21 +11699,51 @@ async function ensureIndexFresh(rootDir, options = {}) {
       },
       getIntrospection: (filepath) => introspection.getFile(filepath)
     };
-    const totalFiles = currentFiles.length;
-    let completedCount = 0;
-    const processIncrementalFile = async (filepath) => {
+    const statCheck = async (filepath) => {
       const relativePath = path21.relative(rootDir, filepath);
       try {
         const stats = await fs8.stat(filepath);
         const lastModified = stats.mtime.toISOString();
         const existingEntry = manifest.files[relativePath];
-        if (existingEntry && existingEntry.lastModified === lastModified) {
-          completedCount++;
-          return { relativePath, status: "unchanged" };
+        if (!existingEntry) {
+          return { filepath, relativePath, lastModified, needsCheck: true, isNew: true };
+        }
+        if (existingEntry.lastModified === lastModified) {
+          return { filepath, relativePath, lastModified, needsCheck: false, isNew: false };
         }
+        return { filepath, relativePath, lastModified, needsCheck: true, isNew: false };
+      } catch {
+        return null;
+      }
+    };
+    const statCheckStart = Date.now();
+    const statResults = await parallelMap(currentFiles, statCheck, STAT_CONCURRENCY);
+    statCheckMs += Date.now() - statCheckStart;
+    filesStatChecked += currentFiles.length;
+    const filesToProcess = [];
+    let unchangedCount = 0;
+    for (const result2 of statResults) {
+      if (!result2.success || !result2.value)
+        continue;
+      if (result2.value.needsCheck) {
+        filesToProcess.push(result2.value);
+      } else {
+        unchangedCount++;
+      }
+    }
+    if (filesToProcess.length === 0) {
+      totalUnchanged += unchangedCount;
+      continue;
+    }
+    let completedCount = 0;
+    const totalToProcess = filesToProcess.length;
+    const processChangedFile = async (statResult) => {
+      const { filepath, relativePath, lastModified, isNew } = statResult;
+      try {
         const content = await fs8.readFile(filepath, "utf-8");
         const contentHash = computeContentHash(content);
-        if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+        const existingEntry = manifest.files[relativePath];
+        if (!isNew && existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
           completedCount++;
           return {
             relativePath,
@@ -11672,7 +11753,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
           };
         }
         completedCount++;
-        logger.progress(`  [${completedCount}/${totalFiles}] Indexing: ${relativePath}`);
+        logger.progress(`  [${completedCount}/${totalToProcess}] Indexing: ${relativePath}`);
         introspection.addFile(relativePath, content);
         const fileIndex = await module2.indexFile(relativePath, content, ctx);
         if (!fileIndex) {
@@ -11691,8 +11772,12 @@ async function ensureIndexFresh(rootDir, options = {}) {
         return { relativePath, status: "error", error };
       }
     };
+    const indexingStart = Date.now();
     const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
-    const results = await parallelMap(currentFiles, processIncrementalFile, concurrency);
+    const results = await parallelMap(filesToProcess, processChangedFile, concurrency);
+    indexingMs += Date.now() - indexingStart;
+    filesIndexed += filesToProcess.length;
+    totalUnchanged += unchangedCount;
     logger.clearProgress();
     let mtimeUpdates = 0;
     for (const item of results) {
@@ -11746,12 +11831,38 @@ async function ensureIndexFresh(rootDir, options = {}) {
   }
   if (totalIndexed > 0 || totalRemoved > 0) {
     await updateGlobalManifest(rootDir, enabledModules, config);
+    clearFreshnessCache();
   }
-  return {
+  const result = {
     indexed: totalIndexed,
     removed: totalRemoved,
     unchanged: totalUnchanged
   };
+  if (showTiming) {
+    result.timing = {
+      totalMs: Date.now() - startTime,
+      fileDiscoveryMs,
+      statCheckMs,
+      indexingMs,
+      cleanupMs,
+      filesDiscovered,
+      filesStatChecked,
+      filesIndexed,
+      fromCache: false
+    };
+  }
+  let finalManifestMtime = currentManifestMtime;
+  try {
+    const manifestStats = await fs8.stat(globalManifestPath);
+    finalManifestMtime = manifestStats.mtimeMs;
+  } catch {}
+  freshnessCache = {
+    rootDir,
+    result: { indexed: totalIndexed, removed: totalRemoved, unchanged: totalUnchanged },
+    timestamp: Date.now(),
+    manifestMtime: finalManifestMtime
+  };
+  return result;
 }
 async function indexWithModule(rootDir, files, module2, config, verbose, introspection, logger, concurrency = DEFAULT_CONCURRENCY) {
   const result = {
@@ -11889,16 +12000,13 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
 async function findFiles(rootDir, config) {
   const patterns = config.extensions.map((ext) => `**/*${ext}`);
   const ignorePatterns = config.ignorePaths.map((p) => `**/${p}/**`);
-  const files = [];
-  for (const pattern of patterns) {
-    const matches = await glob(pattern, {
-      cwd: rootDir,
-      absolute: true,
-      ignore: ignorePatterns
-    });
-    files.push(...matches);
-  }
-  return [...new Set(files)];
+  const results = await Promise.all(patterns.map((pattern) => glob(pattern, {
+    cwd: rootDir,
+    absolute: true,
+    ignore: ignorePatterns
+  })));
+  const allFiles = results.flat();
+  return [...new Set(allFiles)];
 }
 async function loadModuleManifest(rootDir, moduleId, config) {
   const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
@@ -13476,4 +13584,4 @@ export {
   ConsoleLogger
 };
 
-//# debugId=7E760B022D4CA40A64756E2164756E21
+//# debugId=EA9D4B791B6DE73764756E2164756E21