raggrep 0.10.2 → 0.10.4

package/dist/app/indexer/index.d.ts

@@ -23,6 +23,29 @@ export interface IndexOptions {
     logger?: Logger;
     /** Number of files to process in parallel (default: auto based on CPU cores) */
     concurrency?: number;
+    /** Show timing information for each stage */
+    timing?: boolean;
+}
+/** Timing information for performance profiling */
+export interface TimingInfo {
+    /** Total time in milliseconds */
+    totalMs: number;
+    /** Time spent on file discovery (glob) */
+    fileDiscoveryMs: number;
+    /** Time spent on stat checks */
+    statCheckMs: number;
+    /** Time spent on indexing changed files */
+    indexingMs: number;
+    /** Time spent on cleanup operations */
+    cleanupMs: number;
+    /** Number of files discovered */
+    filesDiscovered: number;
+    /** Number of files that needed stat check */
+    filesStatChecked: number;
+    /** Number of files that needed indexing */
+    filesIndexed: number;
+    /** Whether result was from cache */
+    fromCache: boolean;
 }
 export interface EnsureFreshResult {
     /** Number of files indexed (new or modified) */
@@ -31,6 +54,8 @@ export interface EnsureFreshResult {
     removed: number;
     /** Number of files unchanged (used cache) */
     unchanged: number;
+    /** Timing information (only present if timing option was enabled) */
+    timing?: TimingInfo;
 }
 export interface CleanupResult {
     moduleId: string;

package/dist/cli/main.js

@@ -11545,6 +11545,15 @@ async function resetIndex(rootDir) {
 async function ensureIndexFresh(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
+  const showTiming = options.timing ?? false;
+  const startTime = Date.now();
+  let fileDiscoveryMs = 0;
+  let statCheckMs = 0;
+  let indexingMs = 0;
+  let cleanupMs = 0;
+  let filesDiscovered = 0;
+  let filesStatChecked = 0;
+  let filesIndexed = 0;
   const logger = options.logger ? options.logger : quiet ? createSilentLogger() : createLogger({ verbose });
   rootDir = path22.resolve(rootDir);
   const status = await getIndexStatus(rootDir);
@@ -11576,16 +11585,35 @@ async function ensureIndexFresh(rootDir, options = {}) {
   const now = Date.now();
   if (freshnessCache && freshnessCache.rootDir === rootDir && now - freshnessCache.timestamp < FRESHNESS_CACHE_TTL_MS && freshnessCache.manifestMtime === currentManifestMtime) {
     logger.debug("Using cached freshness check result");
-    return freshnessCache.result;
+    const cachedResult = { ...freshnessCache.result };
+    if (showTiming) {
+      cachedResult.timing = {
+        totalMs: Date.now() - startTime,
+        fileDiscoveryMs: 0,
+        statCheckMs: 0,
+        indexingMs: 0,
+        cleanupMs: 0,
+        filesDiscovered: 0,
+        filesStatChecked: 0,
+        filesIndexed: 0,
+        fromCache: true
+      };
+    }
+    return cachedResult;
   }
   await registerBuiltInModules();
   const enabledModules = registry.getEnabled(config);
   if (enabledModules.length === 0) {
     return { indexed: 0, removed: 0, unchanged: 0 };
   }
+  const fileDiscoveryStart = Date.now();
   const introspection = new IntrospectionIndex(rootDir);
-  await introspection.initialize();
-  const currentFiles = await findFiles(rootDir, config);
+  const [, currentFiles] = await Promise.all([
+    introspection.initialize(),
+    findFiles(rootDir, config)
+  ]);
+  fileDiscoveryMs = Date.now() - fileDiscoveryStart;
+  filesDiscovered = currentFiles.length;
   const currentFileSet = new Set(currentFiles.map((f) => path22.relative(rootDir, f)));
   let totalIndexed = 0;
   let totalRemoved = 0;
@@ -11614,20 +11642,21 @@ async function ensureIndexFresh(rootDir, options = {}) {
         filesToRemove.push(filepath);
       }
     }
+    const cleanupStart = Date.now();
     const removedFilepaths = [];
-    for (const filepath of filesToRemove) {
-      logger.debug(`  Removing stale: ${filepath}`);
-      const indexFilePath = path22.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
-      try {
-        await fs8.unlink(indexFilePath);
-      } catch {}
-      const symbolicFilePath = path22.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
-      try {
-        await fs8.unlink(symbolicFilePath);
-      } catch {}
-      delete manifest.files[filepath];
-      removedFilepaths.push(filepath);
-      totalRemoved++;
+    if (filesToRemove.length > 0) {
+      await Promise.all(filesToRemove.map(async (filepath) => {
+        logger.debug(`  Removing stale: ${filepath}`);
+        const indexFilePath = path22.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
+        const symbolicFilePath = path22.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
+        await Promise.all([
+          fs8.unlink(indexFilePath).catch(() => {}),
+          fs8.unlink(symbolicFilePath).catch(() => {})
+        ]);
+        delete manifest.files[filepath];
+        removedFilepaths.push(filepath);
+      }));
+      totalRemoved += removedFilepaths.length;
     }
     if (removedFilepaths.length > 0) {
       try {
@@ -11641,6 +11670,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
         await literalIndex.save();
       } catch {}
     }
+    cleanupMs += Date.now() - cleanupStart;
     const ctx = {
       rootDir,
       config,
@@ -11655,21 +11685,51 @@ async function ensureIndexFresh(rootDir, options = {}) {
       },
       getIntrospection: (filepath) => introspection.getFile(filepath)
     };
-    const totalFiles = currentFiles.length;
-    let completedCount = 0;
-    const processIncrementalFile = async (filepath) => {
+    const statCheck = async (filepath) => {
       const relativePath = path22.relative(rootDir, filepath);
       try {
         const stats = await fs8.stat(filepath);
         const lastModified = stats.mtime.toISOString();
         const existingEntry = manifest.files[relativePath];
-        if (existingEntry && existingEntry.lastModified === lastModified) {
-          completedCount++;
-          return { relativePath, status: "unchanged" };
+        if (!existingEntry) {
+          return { filepath, relativePath, lastModified, needsCheck: true, isNew: true };
         }
+        if (existingEntry.lastModified === lastModified) {
+          return { filepath, relativePath, lastModified, needsCheck: false, isNew: false };
+        }
+        return { filepath, relativePath, lastModified, needsCheck: true, isNew: false };
+      } catch {
+        return null;
+      }
+    };
+    const statCheckStart = Date.now();
+    const statResults = await parallelMap(currentFiles, statCheck, STAT_CONCURRENCY);
+    statCheckMs += Date.now() - statCheckStart;
+    filesStatChecked += currentFiles.length;
+    const filesToProcess = [];
+    let unchangedCount = 0;
+    for (const result2 of statResults) {
+      if (!result2.success || !result2.value)
+        continue;
+      if (result2.value.needsCheck) {
+        filesToProcess.push(result2.value);
+      } else {
+        unchangedCount++;
+      }
+    }
+    if (filesToProcess.length === 0) {
+      totalUnchanged += unchangedCount;
+      continue;
+    }
+    let completedCount = 0;
+    const totalToProcess = filesToProcess.length;
+    const processChangedFile = async (statResult) => {
+      const { filepath, relativePath, lastModified, isNew } = statResult;
+      try {
         const content = await fs8.readFile(filepath, "utf-8");
         const contentHash = computeContentHash(content);
-        if (existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
+        const existingEntry = manifest.files[relativePath];
+        if (!isNew && existingEntry?.contentHash && existingEntry.contentHash === contentHash) {
           completedCount++;
           return {
             relativePath,
@@ -11679,7 +11739,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
           };
         }
         completedCount++;
-        logger.progress(`  [${completedCount}/${totalFiles}] Indexing: ${relativePath}`);
+        logger.progress(`  [${completedCount}/${totalToProcess}] Indexing: ${relativePath}`);
         introspection.addFile(relativePath, content);
         const fileIndex = await module2.indexFile(relativePath, content, ctx);
         if (!fileIndex) {
@@ -11698,8 +11758,12 @@ async function ensureIndexFresh(rootDir, options = {}) {
         return { relativePath, status: "error", error };
       }
     };
+    const indexingStart = Date.now();
     const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
-    const results = await parallelMap(currentFiles, processIncrementalFile, concurrency);
+    const results = await parallelMap(filesToProcess, processChangedFile, concurrency);
+    indexingMs += Date.now() - indexingStart;
+    filesIndexed += filesToProcess.length;
+    totalUnchanged += unchangedCount;
     logger.clearProgress();
     let mtimeUpdates = 0;
     for (const item of results) {
@@ -11760,6 +11824,19 @@ async function ensureIndexFresh(rootDir, options = {}) {
     removed: totalRemoved,
     unchanged: totalUnchanged
   };
+  if (showTiming) {
+    result.timing = {
+      totalMs: Date.now() - startTime,
+      fileDiscoveryMs,
+      statCheckMs,
+      indexingMs,
+      cleanupMs,
+      filesDiscovered,
+      filesStatChecked,
+      filesIndexed,
+      fromCache: false
+    };
+  }
   let finalManifestMtime = currentManifestMtime;
   try {
     const manifestStats = await fs8.stat(globalManifestPath);
@@ -11767,7 +11844,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
   } catch {}
   freshnessCache = {
     rootDir,
-    result,
+    result: { indexed: totalIndexed, removed: totalRemoved, unchanged: totalUnchanged },
     timestamp: Date.now(),
     manifestMtime: finalManifestMtime
   };
@@ -11909,16 +11986,13 @@ async function indexWithModule(rootDir, files, module2, config, verbose, introsp
 async function findFiles(rootDir, config) {
   const patterns = config.extensions.map((ext) => `**/*${ext}`);
   const ignorePatterns = config.ignorePaths.map((p) => `**/${p}/**`);
-  const files = [];
-  for (const pattern of patterns) {
-    const matches = await glob(pattern, {
-      cwd: rootDir,
-      absolute: true,
-      ignore: ignorePatterns
-    });
-    files.push(...matches);
-  }
-  return [...new Set(files)];
+  const results = await Promise.all(patterns.map((pattern) => glob(pattern, {
+    cwd: rootDir,
+    absolute: true,
+    ignore: ignorePatterns
+  })));
+  const allFiles = results.flat();
+  return [...new Set(allFiles)];
 }
 async function loadModuleManifest(rootDir, moduleId, config) {
   const manifestPath = getModuleManifestPath(rootDir, moduleId, config);
@@ -12087,7 +12161,7 @@ async function getIndexStatus(rootDir) {
   }
   return status;
 }
-var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.0.0", DEFAULT_CONCURRENCY;
+var FRESHNESS_CACHE_TTL_MS = 5000, freshnessCache = null, INDEX_SCHEMA_VERSION = "2.0.0", DEFAULT_CONCURRENCY, STAT_CONCURRENCY;
 var init_indexer = __esm(() => {
   init_config2();
   init_registry();
@@ -12095,6 +12169,7 @@ var init_indexer = __esm(() => {
   init_logger();
   init_watcher();
   DEFAULT_CONCURRENCY = getOptimalConcurrency();
+  STAT_CONCURRENCY = Math.max(32, getOptimalConcurrency() * 4);
 });
 
 // node_modules/balanced-match/index.js
@@ -13668,7 +13743,7 @@ init_logger();
 // package.json
 var package_default = {
   name: "raggrep",
-  version: "0.10.2",
+  version: "0.10.4",
   description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   type: "module",
   main: "./dist/index.js",
@@ -13765,6 +13840,7 @@ function parseFlags(args3) {
     help: false,
     verbose: false,
     watch: false,
+    timing: false,
     remaining: []
   };
   for (let i2 = 0;i2 < args3.length; i2++) {
@@ -13775,6 +13851,8 @@ function parseFlags(args3) {
       flags2.verbose = true;
     } else if (arg === "--watch" || arg === "-w") {
       flags2.watch = true;
+    } else if (arg === "--timing" || arg === "-T") {
+      flags2.timing = true;
     } else if (arg === "--model" || arg === "-m") {
       const modelName = args3[++i2];
       if (modelName && modelName in EMBEDDING_MODELS) {
@@ -13933,6 +14011,7 @@ Options:
   -s, --min-score <n>  Minimum similarity score 0-1 (default: 0.15)
   -t, --type <ext>     Filter by file extension (e.g., ts, tsx, js)
   -f, --filter <path>  Filter by path or glob pattern (can be used multiple times)
+  -T, --timing         Show timing breakdown for performance profiling
   -h, --help           Show this help message
 
 Note:
@@ -13986,7 +14065,8 @@ Examples:
         const freshStats = await ensureIndexFresh2(process.cwd(), {
           model: flags2.model,
           quiet: true,
-          logger: silentLogger
+          logger: silentLogger,
+          timing: flags2.timing
         });
         console.log("RAGgrep Search");
         console.log(`==============
@@ -14003,6 +14083,23 @@ Examples:
 `);
         } else {
           console.log(`Using cached index (no changes detected).
+`);
+        }
+        if (flags2.timing && freshStats.timing) {
+          const t = freshStats.timing;
+          console.log("┌─ Timing ─────────────────────────────────────┐");
+          if (t.fromCache) {
+            console.log(`│  Cache hit (TTL-based)                       │`);
+            console.log(`│  Total: ${t.totalMs.toFixed(0).padStart(6)}ms                              │`);
+          } else {
+            console.log(`│  File discovery: ${t.fileDiscoveryMs.toFixed(0).padStart(6)}ms (${t.filesDiscovered} files)`.padEnd(47) + "│");
+            console.log(`│  Stat checks:    ${t.statCheckMs.toFixed(0).padStart(6)}ms (${t.filesStatChecked} files)`.padEnd(47) + "│");
+            console.log(`│  Indexing:       ${t.indexingMs.toFixed(0).padStart(6)}ms (${t.filesIndexed} files)`.padEnd(47) + "│");
+            console.log(`│  Cleanup:        ${t.cleanupMs.toFixed(0).padStart(6)}ms`.padEnd(47) + "│");
+            console.log(`│  ─────────────────────────────────────────── │`);
+            console.log(`│  Total:          ${t.totalMs.toFixed(0).padStart(6)}ms`.padEnd(47) + "│");
+          }
+          console.log(`└──────────────────────────────────────────────┘
 `);
         }
         const filePatterns = flags2.fileType ? [`*.${flags2.fileType}`] : undefined;
@@ -14262,4 +14359,4 @@ Run 'raggrep <command> --help' for more information.
 }
 main();
 
-//# debugId=04D75A20D6C87C2264756E2164756E21
+//# debugId=5A4116C9AF3188E264756E2164756E21