raggrep 0.10.1 → 0.10.4

package/dist/app/indexer/index.d.ts

@@ -1,4 +1,9 @@
 import type { EmbeddingModelName, Logger } from "../../domain/ports";
+/**
+ * Clear the freshness cache.
+ * Call this after explicit indexing operations.
+ */
+export declare function clearFreshnessCache(): void;
 export interface IndexResult {
     moduleId: string;
     indexed: number;
@@ -18,6 +23,29 @@ export interface IndexOptions {
     logger?: Logger;
     /** Number of files to process in parallel (default: auto based on CPU cores) */
     concurrency?: number;
+    /** Show timing information for each stage */
+    timing?: boolean;
+}
+/** Timing information for performance profiling */
+export interface TimingInfo {
+    /** Total time in milliseconds */
+    totalMs: number;
+    /** Time spent on file discovery (glob) */
+    fileDiscoveryMs: number;
+    /** Time spent on stat checks */
+    statCheckMs: number;
+    /** Time spent on indexing changed files */
+    indexingMs: number;
+    /** Time spent on cleanup operations */
+    cleanupMs: number;
+    /** Number of files discovered */
+    filesDiscovered: number;
+    /** Number of files that needed stat check */
+    filesStatChecked: number;
+    /** Number of files that needed indexing */
+    filesIndexed: number;
+    /** Whether result was from cache */
+    fromCache: boolean;
 }
 export interface EnsureFreshResult {
     /** Number of files indexed (new or modified) */
@@ -26,6 +54,8 @@ export interface EnsureFreshResult {
     removed: number;
     /** Number of files unchanged (used cache) */
     unchanged: number;
+    /** Timing information (only present if timing option was enabled) */
+    timing?: TimingInfo;
 }
 export interface CleanupResult {
     moduleId: string;

package/dist/cli/main.js

@@ -11235,6 +11235,7 @@ async function watchDirectory(rootDir, options = {}) {
     isIndexing = true;
     const changes = new Map(pendingChanges);
     pendingChanges.clear();
+    clearFreshnessCache();
     try {
       const filesToIndex = [];
       const filesToDelete = [];
@@ -11377,6 +11378,7 @@ __export(exports_indexer, {
   indexDirectory: () => indexDirectory,
   getIndexStatus: () => getIndexStatus,
   ensureIndexFresh: () => ensureIndexFresh,
+  clearFreshnessCache: () => clearFreshnessCache,
   cleanupIndex: () => cleanupIndex
 });
 import { glob } from "glob";
@@ -11384,6 +11386,9 @@ import * as fs8 from "fs/promises";
 import * as path22 from "path";
 import * as os3 from "os";
 import * as crypto2 from "crypto";
+function clearFreshnessCache() {
+  freshnessCache = null;
+}
 function computeContentHash(content) {
   return crypto2.createHash("sha256").update(content, "utf-8").digest("hex");
 }
@@ -11427,6 +11432,7 @@ async function indexDirectory(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
   const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+  clearFreshnessCache();
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path22.resolve(rootDir);
   const location = getIndexLocation(rootDir);
@@ -11525,6 +11531,7 @@ async function deleteIndex(rootDir) {
 }
 async function resetIndex(rootDir) {
   rootDir = path22.resolve(rootDir);
+  clearFreshnessCache();
   const status = await getIndexStatus(rootDir);
   if (!status.exists) {
     throw new Error(`No index found for ${rootDir}`);
@@ -11538,10 +11545,20 @@ async function resetIndex(rootDir) {
 async function ensureIndexFresh(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
+  const showTiming = options.timing ?? false;
+  const startTime = Date.now();
+  let fileDiscoveryMs = 0;
+  let statCheckMs = 0;
+  let indexingMs = 0;
+  let cleanupMs = 0;
+  let filesDiscovered = 0;
+  let filesStatChecked = 0;
+  let filesIndexed = 0;
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path22.resolve(rootDir);
   const status = await getIndexStatus(rootDir);
   if (!status.exists) {
+    clearFreshnessCache();
     logger.info(`No index found. Creating index...
 `);
     const results = await indexDirectory(rootDir, { ...options, logger });
@@ -11550,6 +11567,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
   }
   const versionCompatible = await isIndexVersionCompatible(rootDir);
   if (!versionCompatible) {
+    clearFreshnessCache();
     logger.info(`Index version incompatible. Rebuilding...
 `);
     await deleteIndex(rootDir);
@@ -11558,14 +11576,44 @@ async function ensureIndexFresh(rootDir, options = {}) {
     return { indexed: totalIndexed2, removed: 0, unchanged: 0 };
   }
   const config = await loadConfig(rootDir);
+  const globalManifestPath = getGlobalManifestPath(rootDir, config);
+  let currentManifestMtime = 0;
+  try {
+    const manifestStats = await fs8.stat(globalManifestPath);
+    currentManifestMtime = manifestStats.mtimeMs;
+  } catch {}
+  const now = Date.now();
+  if (freshnessCache && freshnessCache.rootDir === rootDir && now - freshnessCache.timestamp < FRESHNESS_CACHE_TTL_MS && freshnessCache.manifestMtime === currentManifestMtime) {
+    logger.debug("Using cached freshness check result");
+    const cachedResult = { ...freshnessCache.result };
+    if (showTiming) {
+      cachedResult.timing = {
+        totalMs: Date.now() - startTime,
+        fileDiscoveryMs: 0,
+        statCheckMs: 0,
+        indexingMs: 0,
+        cleanupMs: 0,
+        filesDiscovered: 0,
+        filesStatChecked: 0,
+        filesIndexed: 0,
+        fromCache: true
+      };
+    }
+    return cachedResult;
+  }
   await registerBuiltInModules();
   const enabledModules = registry.getEnabled(config);
   if (enabledModules.length === 0) {
     return { indexed: 0, removed: 0, unchanged: 0 };
   }
+  const fileDiscoveryStart = Date.now();
   const introspection = new IntrospectionIndex(rootDir);
-  await introspection.initialize();
-  const currentFiles = await findFiles(rootDir, config);
+  const [, currentFiles] = await Promise.all([
+    introspection.initialize(),
+    findFiles(rootDir, config)
+  ]);
+  fileDiscoveryMs = Date.now() - fileDiscoveryStart;
+  filesDiscovered = currentFiles.length;
   const currentFileSet = new Set(currentFiles.map((f) => path22.relative(rootDir, f)));
   let totalIndexed = 0;
   let totalRemoved = 0;
@@ -11594,20 +11642,21 @@ async function ensureIndexFresh(rootDir, options = {}) {
         filesToRemove.push(filepath);
       }
     }
+    const cleanupStart = Date.now();
     const removedFilepaths = [];
-    for (const filepath of filesToRemove) {
-      logger.debug(`  Removing stale: ${filepath}`);
-      const indexFilePath = path22.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
-      try {
-        await fs8.unlink(indexFilePath);
-      } catch {}
-      const symbolicFilePath = path22.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
-      try {
-        await fs8.unlink(symbolicFilePath);
-      } catch {}
-      delete manifest.files[filepath];
-      removedFilepaths.push(filepath);
-      totalRemoved++;
+    if (filesToRemove.length > 0) {
+      await Promise.all(filesToRemove.map(async (filepath) => {
+        logger.debug(`  Removing stale: ${filepath}`);
+        const indexFilePath = path22.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
+        const symbolicFilePath = path22.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
+        await Promise.all([
+          fs8.unlink(indexFilePath).catch(() => {}),
+          fs8.unlink(symbolicFilePath).catch(() => {})
+        ]);
+        delete manifest.files[filepath];
+        removedFilepaths.push(filepath);
+      }));
+      totalRemoved += removedFilepaths.length;
     }
     if (removedFilepaths.length > 0) {
       try {
@@ -11621,6 +11670,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
         await literalIndex.save();
       } catch {}
     }
+    cleanupMs += Date.now() - cleanupStart;
     const ctx = {
       rootDir,
       config,
@@ -11635,21 +11685,51 @@ async function ensureIndexFresh(rootDir, options = {}) {
       },
       getIntrospection: (filepath) => introspection.getFile(filepath)
     };
-    const totalFiles = currentFiles.length;
-    let completedCount = 0;
-    const processIncrementalFile = async (filepath) => {
+    const statCheck = async (filepath) => {
       const relativePath = path22.relative(rootDir, filepath);
       try {
         const stats = await fs8.stat(filepath);
         const lastModified = stats.mtime.toISOString();
         const existingEntry = manifest.files[relativePath];
-        if (existingEntry && existingEntry.lastModified === lastModified) {
-          completedCount++;
-          return { relativePath, status: "unchanged" };
+        if (!existingEntry) {
+          return { filepath, relativePath, lastModified, needsCheck: true, isNew: true };
         }
+        if (existingEntry.lastModified === lastModified) {
+          return { filepath, relativePath, lastModified, needsCheck: false, isNew: false };
+        }
+        return { filepath, relativePath, lastModified, needsCheck: true, isNew: false };
+      } catch {
+        return null;
+      }
+    };
+    const statCheckStart = Date.now();
+    const statResults = await parallelMap(currentFiles, statCheck, STAT_CONCURRENCY);
+    statCheckMs += Date.now() - statCheckStart;
+    filesStatChecked += currentFiles.length;
+    const filesToProcess = [];
+    let unchangedCount = 0;
+    for (const result2 of statResults) {
+      if (!result2.success || !result2.value)
+        continue;
+      if (result2.value.needsCheck) {
+        filesToProcess.push(result2.value);
+      } else {
+        unchangedCount++;
+      }
+    }
+    if (filesToProcess.length === 0) {
+      totalUnchanged += unchangedCount;
+      continue;
+    }
+    let completedCount = 0;
+    const totalToProcess = filesToProcess.length;
+    const processChangedFile = async (statResult) => {
+      const { filepath, relativePath, lastModified, isNew } = statResult;
+      try {
         const content = await fs8.readFile(filepath, "utf-8");
         const contentHash = computeContentHash(content);
-        if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+        const existingEntry = manifest.files[relativePath];
+        if (!isNew && existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
           completedCount++;
           return {
             relativePath,
@@ -11659,7 +11739,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
           };
         }
         completedCount++;
-        logger.progress(`  [${completedCount}/${totalFiles}] Indexing: ${relativePath}`);
+        logger.progress(`  [${completedCount}/${totalToProcess}] Indexing: ${relativePath}`);
         introspection.addFile(relativePath, content);
         const fileIndex = await module2.indexFile(relativePath, content, ctx);
         if (!fileIndex) {
@@ -11678,8 +11758,12 @@ async function ensureIndexFresh(rootDir, options = {}) {
         return { relativePath, status: "error", error };
       }
     };
+    const indexingStart = Date.now();
     const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
-    const results = await parallelMap(currentFiles, processIncrementalFile, concurrency);
+    const results = await parallelMap(filesToProcess, processChangedFile, concurrency);
+    indexingMs += Date.now() - indexingStart;
+    filesIndexed += filesToProcess.length;
+    totalUnchanged += unchangedCount;
     logger.clearProgress();
     let mtimeUpdates = 0;
     for (const item of results) {
@@ -11733,12 +11817,38 @@ async function ensureIndexFresh(rootDir, options = {}) {
   }
   if (totalIndexed > 0 || totalRemoved > 0) {
     await updateGlobalManifest(rootDir, enabledModules, config);
+    clearFreshnessCache();
   }
-  return {
+  const result = {
     indexed: totalIndexed,
     removed: totalRemoved,
     unchanged: totalUnchanged
   };
+  if (showTiming) {
+    result.timing = {
+      totalMs: Date.now() - startTime,
+      fileDiscoveryMs,
+      statCheckMs,
+      indexingMs,
+      cleanupMs,
+      filesDiscovered,
+      filesStatChecked,
+      filesIndexed,
+      fromCache: false
+    };
+  }
+  let finalManifestMtime = currentManifestMtime;
+  try {
+    const manifestStats = await fs8.stat(globalManifestPath);
+    finalManifestMtime = manifestStats.mtimeMs;
+  } catch {}
+  freshnessCache = {
+    rootDir,
+    result: { indexed: totalIndexed, removed: totalRemoved, unchanged: totalUnchanged },
+    timestamp: Date.now(),
+    manifestMtime: finalManifestMtime
+  };
+  return result;
 }
 async function indexWithModule(rootDir, files, module2, config, verbose, introspection, logger, concurrency = DEFAULT_CONCURRENCY) {
   const result = {
@@ -11876,16 +11986,13 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
 async function findFiles(rootDir, config) {
   const patterns = config.extensions.map((ext) => `**/*${ext}`);
   const ignorePatterns = config.ignorePaths.map((p) => `**/${p}/**`);
-  const files = [];
-  for (const pattern of patterns) {
-    const matches = await glob(pattern, {
-      cwd: rootDir,
-      absolute: true,
-      ignore: ignorePatterns
-    });
-    files.push(...matches);
-  }
-  return [...new Set(files)];
+  const results = await Promise.all(patterns.map((pattern) => glob(pattern, {
+    cwd: rootDir,
+    absolute: true,
+    ignore: ignorePatterns
+  })));
+  const allFiles = results.flat();
+  return [...new Set(allFiles)];
 }
 async function loadModuleManifest(rootDir, moduleId, config) {
   const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
@@ -12054,7 +12161,7 @@ async function getIndexStatus(rootDir) {
   }
   return status;
 }
-var INDEX_SCHEMA_VERSION = "2.0.0", DEFAULT_CONCURRENCY;
+var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.0.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY;
 var init_indexer = __esm(() => {
   init_config2();
   init_registry();
@@ -12062,6 +12169,7 @@ var init_indexer = __esm(() => {
   init_logger();
   init_watcher();
   DEFAULT_CONCURRENCY = getOptimalConcurrency();
+  STAT_CONCURRENCY = Math.max(32, getOptimalConcurrency() * 4);
 });
 
 // node_modules/balanced-match/index.js
@@ -13635,7 +13743,7 @@ init_logger();
 // package.json
 var package_default = {
   name: "raggrep",
-  version: "0.10.1",
+  version: "0.10.4",
   description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   type: "module",
   main: "./dist/index.js",
@@ -13732,6 +13840,7 @@ function parseFlags(args3) {
     help: false,
     verbose: false,
     watch: false,
+    timing: false,
     remaining: []
   };
   for (let i2 = 0;i2 < args3.length; i2++) {
@@ -13742,6 +13851,8 @@ function parseFlags(args3) {
       flags2.verbose = true;
     } else if (arg === "--watch" || arg === "-w") {
       flags2.watch = true;
+    } else if (arg === "--timing" || arg === "-T") {
+      flags2.timing = true;
     } else if (arg === "--model" || arg === "-m") {
       const modelName = args3[++i2];
       if (modelName && modelName in EMBEDDING_MODELS) {
@@ -13900,6 +14011,7 @@ Options:
   -s, --min-score <n>  Minimum similarity score 0-1 (default: 0.15)
   -t, --type <ext>     Filter by file extension (e.g., ts, tsx, js)
   -f, --filter <path>  Filter by path or glob pattern (can be used multiple times)
+  -T, --timing         Show timing breakdown for performance profiling
   -h, --help           Show this help message
 
 Note:
@@ -13953,7 +14065,8 @@ Examples:
         const freshStats = await ensureIndexFresh2(process.cwd(), {
           model: flags2.model,
           quiet: true,
-          logger: silentLogger
+          logger: silentLogger,
+          timing: flags2.timing
         });
         console.log("RAGgrep Search");
         console.log(`==============
@@ -13970,6 +14083,23 @@ Examples:
 `);
         } else {
           console.log(`Using cached index (no changes detected).
+`);
+        }
+        if (flags2.timing && freshStats.timing) {
+          const t = freshStats.timing;
+          console.log("┌─ Timing ─────────────────────────────────────┐");
+          if (t.fromCache) {
+            console.log(`│  Cache hit (TTL-based)                       │`);
+            console.log(`│  Total: ${t.totalMs.toFixed(0).padStart(6)}ms                              │`);
+          } else {
+            console.log(`│  File discovery: ${t.fileDiscoveryMs.toFixed(0).padStart(6)}ms (${t.filesDiscovered} files)`.padEnd(47) + "│");
+            console.log(`│  Stat checks:    ${t.statCheckMs.toFixed(0).padStart(6)}ms (${t.filesStatChecked} files)`.padEnd(47) + "│");
+            console.log(`│  Indexing:       ${t.indexingMs.toFixed(0).padStart(6)}ms (${t.filesIndexed} files)`.padEnd(47) + "│");
+            console.log(`│  Cleanup:        ${t.cleanupMs.toFixed(0).padStart(6)}ms`.padEnd(47) + "│");
+            console.log(`│  ─────────────────────────────────────────── │`);
+            console.log(`│  Total:          ${t.totalMs.toFixed(0).padStart(6)}ms`.padEnd(47) + "│");
+          }
+          console.log(`└──────────────────────────────────────────────┘
 `);
         }
         const filePatterns = flags2.fileType ? [`*.${flags2.fileType}`] : undefined;
@@ -14229,4 +14359,4 @@ Run 'raggrep <command> --help' for more information.
 }
 main();
 
-//# debugId=983FE42C702490DF64756E2164756E21
+//# debugId=5A4116C9AF3188E264756E2164756E21