raggrep 0.5.0 → 0.5.2

package/README.md

@@ -78,19 +78,60 @@ This monitors file changes and re-indexes automatically. Useful during active de
 [Watch] language/typescript: 2 indexed, 0 errors
 ```
 
-## CLI Quick Reference
+## CLI Reference
+
+### Commands
 
 ```bash
-# Search (auto-indexes if needed)
-raggrep query "user login"
-raggrep query "error handling" --top 5
-raggrep query "database" --type ts
+raggrep query <query>    # Search the codebase
+raggrep index            # Build/update the index
+raggrep status           # Show index status
+raggrep reset            # Clear the index
+```
 
-# Watch mode
-raggrep index --watch
+### Query Options
+
+```bash
+raggrep query "user login"                    # Basic search
+raggrep query "error handling" --top 5        # Limit results
+raggrep query "database" --min-score 0.2      # Set minimum score threshold
+raggrep query "interface" --type ts           # Filter by file extension
+raggrep query "auth" --filter src/auth        # Filter by path
+raggrep query "api" -f src/api -f src/routes  # Multiple path filters
+```
+
+| Flag | Short | Description |
+|------|-------|-------------|
+| `--top <n>` | `-k` | Number of results to return (default: 10) |
+| `--min-score <n>` | `-s` | Minimum similarity score 0-1 (default: 0.15) |
+| `--type <ext>` | `-t` | Filter by file extension (e.g., ts, tsx, js) |
+| `--filter <path>` | `-f` | Filter by path prefix (can be used multiple times) |
+| `--help` | `-h` | Show help message |
+
+### Index Options
 
-# Check index status
-raggrep status
+```bash
+raggrep index                        # Index current directory
+raggrep index --watch                # Watch mode - re-index on file changes
+raggrep index --verbose              # Show detailed progress
+raggrep index --concurrency 8        # Set parallel workers (default: auto)
+raggrep index --model bge-small-en-v1.5  # Use specific embedding model
+```
+
+| Flag | Short | Description |
+|------|-------|-------------|
+| `--watch` | `-w` | Watch for file changes and re-index automatically |
+| `--verbose` | `-v` | Show detailed progress |
+| `--concurrency <n>` | `-c` | Number of parallel workers (default: auto based on CPU) |
+| `--model <name>` | `-m` | Embedding model to use |
+| `--help` | `-h` | Show help message |
+
+### Other Commands
+
+```bash
+raggrep status           # Show index status and statistics
+raggrep reset            # Clear the index completely
+raggrep --version        # Show version
 ```
 
 ## How It Works
@@ -104,9 +145,13 @@ The index is stored in a system temp directory, keeping your project clean.
 
 ## What Gets Indexed
 
-**File types:** `.ts`, `.tsx`, `.js`, `.jsx`, `.py`, `.go`, `.rs`, `.java`, `.md`, `.txt`
+**TypeScript/JavaScript:** `.ts`, `.tsx`, `.js`, `.jsx`, `.mjs`, `.cjs` — AST-parsed for functions, classes, interfaces, types, enums
+
+**Documentation:** `.md`, `.txt` — Section-aware parsing with heading extraction
+
+**Data:** `.json` — Structure-aware with key/value extraction
 
-**Code structures:** Functions, classes, interfaces, types, enums, exports
+**Other languages:** `.py`, `.go`, `.rs`, `.java`, `.yaml`, `.yml`, `.toml`, `.sql` — Symbol extraction and keyword search
 
 **Automatically ignored:** `node_modules`, `dist`, `build`, `.git`, and other common directories
 

package/dist/app/indexer/index.d.ts

@@ -4,6 +4,8 @@ export interface IndexResult {
     indexed: number;
     skipped: number;
     errors: number;
+    /** Time taken in milliseconds */
+    durationMs?: number;
 }
 export interface IndexOptions {
     /** Override the embedding model (semantic module) */
@@ -14,7 +16,7 @@ export interface IndexOptions {
     quiet?: boolean;
     /** Logger for progress reporting. If not provided, uses console by default (quiet mode uses silent logger) */
     logger?: Logger;
-    /** Number of files to process in parallel (default: 4) */
+    /** Number of files to process in parallel (default: auto based on CPU cores) */
     concurrency?: number;
 }
 export interface EnsureFreshResult {

package/dist/cli/main.js

@@ -325,6 +325,7 @@ var init_searchResult = __esm(() => {
     topK: 10,
     minScore: 0.15,
     filePatterns: [],
+    pathFilter: [],
     ensureFresh: true
   };
 });
@@ -2087,6 +2088,9 @@ class CoreModule {
   name = "Core Search";
   description = "Language-agnostic text search with symbol extraction";
   version = "1.0.0";
+  supportsFile(_filepath) {
+    return true;
+  }
   symbolIndex = new Map;
   bm25Index = null;
   rootDir = "";
@@ -2979,6 +2983,7 @@ var init_storage = __esm(() => {
 // src/modules/language/typescript/index.ts
 var exports_typescript = {};
 __export(exports_typescript, {
+  supportsFile: () => supportsFile,
   isTypeScriptFile: () => isTypeScriptFile,
   TypeScriptModule: () => TypeScriptModule,
   TYPESCRIPT_EXTENSIONS: () => TYPESCRIPT_EXTENSIONS,
@@ -3017,6 +3022,9 @@ class TypeScriptModule {
   name = "TypeScript Search";
   description = "TypeScript-aware code search with AST parsing and semantic embeddings";
   version = "1.0.0";
+  supportsFile(filepath) {
+    return isTypeScriptFile(filepath);
+  }
   embeddingConfig = null;
   symbolicIndex = null;
   pendingSummaries = new Map;
@@ -3240,7 +3248,7 @@ class TypeScriptModule {
     return references;
   }
 }
-var DEFAULT_MIN_SCORE2 = 0.15, DEFAULT_TOP_K2 = 10, SEMANTIC_WEIGHT = 0.7, BM25_WEIGHT = 0.3, TYPESCRIPT_EXTENSIONS;
+var DEFAULT_MIN_SCORE2 = 0.15, DEFAULT_TOP_K2 = 10, SEMANTIC_WEIGHT = 0.7, BM25_WEIGHT = 0.3, TYPESCRIPT_EXTENSIONS, supportsFile;
 var init_typescript = __esm(() => {
   init_embeddings();
   init_services();
@@ -3257,11 +3265,13 @@ var init_typescript = __esm(() => {
     ".mts",
     ".cts"
   ];
+  supportsFile = isTypeScriptFile;
 });
 
 // src/modules/data/json/index.ts
 var exports_json = {};
 __export(exports_json, {
+  supportsFile: () => supportsFile2,
   isJsonFile: () => isJsonFile,
   JsonModule: () => JsonModule,
   JSON_EXTENSIONS: () => JSON_EXTENSIONS,
@@ -3318,6 +3328,9 @@ class JsonModule {
   name = "JSON Search";
   description = "JSON file search with structure-aware indexing";
   version = "1.0.0";
+  supportsFile(filepath) {
+    return isJsonFile(filepath);
+  }
   embeddingConfig = null;
   symbolicIndex = null;
   pendingSummaries = new Map;
@@ -3479,18 +3492,20 @@ class JsonModule {
     return results.slice(0, topK);
   }
 }
-var DEFAULT_MIN_SCORE3 = 0.15, DEFAULT_TOP_K3 = 10, SEMANTIC_WEIGHT2 = 0.7, BM25_WEIGHT2 = 0.3, JSON_EXTENSIONS;
+var DEFAULT_MIN_SCORE3 = 0.15, DEFAULT_TOP_K3 = 10, SEMANTIC_WEIGHT2 = 0.7, BM25_WEIGHT2 = 0.3, JSON_EXTENSIONS, supportsFile2;
 var init_json = __esm(() => {
   init_embeddings();
   init_services();
   init_config2();
   init_storage();
   JSON_EXTENSIONS = [".json"];
+  supportsFile2 = isJsonFile;
 });
 
 // src/modules/docs/markdown/index.ts
 var exports_markdown = {};
 __export(exports_markdown, {
+  supportsFile: () => supportsFile3,
   isMarkdownFile: () => isMarkdownFile,
   MarkdownModule: () => MarkdownModule,
   MARKDOWN_EXTENSIONS: () => MARKDOWN_EXTENSIONS,
@@ -3596,6 +3611,9 @@ class MarkdownModule {
   name = "Markdown Search";
   description = "Markdown documentation search with section-aware indexing";
   version = "1.0.0";
+  supportsFile(filepath) {
+    return isMarkdownFile(filepath);
+  }
   embeddingConfig = null;
   symbolicIndex = null;
   pendingSummaries = new Map;
@@ -3765,13 +3783,14 @@ ${section.content}` : section.content,
     return results.slice(0, topK);
   }
 }
-var DEFAULT_MIN_SCORE4 = 0.15, DEFAULT_TOP_K4 = 10, SEMANTIC_WEIGHT3 = 0.7, BM25_WEIGHT3 = 0.3, MARKDOWN_EXTENSIONS;
+var DEFAULT_MIN_SCORE4 = 0.15, DEFAULT_TOP_K4 = 10, SEMANTIC_WEIGHT3 = 0.7, BM25_WEIGHT3 = 0.3, MARKDOWN_EXTENSIONS, supportsFile3;
 var init_markdown = __esm(() => {
   init_embeddings();
   init_services();
   init_config2();
   init_storage();
   MARKDOWN_EXTENSIONS = [".md", ".txt"];
+  supportsFile3 = isMarkdownFile;
 });
 
 // src/modules/registry.ts
@@ -4256,6 +4275,7 @@ __export(exports_indexer, {
 import { glob } from "glob";
 import * as fs6 from "fs/promises";
 import * as path15 from "path";
+import * as os3 from "os";
 async function parallelMap(items, processor, concurrency) {
   const results = new Array(items.length);
   let nextIndex = 0;
@@ -4275,6 +4295,23 @@ async function parallelMap(items, processor, concurrency) {
   await Promise.all(workers);
   return results;
 }
+function formatDuration(ms) {
+  if (ms < 1000) {
+    return `${ms}ms`;
+  }
+  const seconds = ms / 1000;
+  if (seconds < 60) {
+    return `${seconds.toFixed(1)}s`;
+  }
+  const minutes = Math.floor(seconds / 60);
+  const remainingSeconds = seconds % 60;
+  return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
+}
+function getOptimalConcurrency() {
+  const cpuCount = os3.cpus().length;
+  const optimal = Math.max(2, Math.min(16, Math.floor(cpuCount * 0.75)));
+  return optimal;
+}
 async function indexDirectory(rootDir, options = {}) {
   const verbose = options.verbose ?? false;
   const quiet = options.quiet ?? false;
@@ -4301,8 +4338,10 @@ async function indexDirectory(rootDir, options = {}) {
   logger.info(`Enabled modules: ${enabledModules.map((m) => m.id).join(", ")}`);
   const files = await findFiles(rootDir, config);
   logger.info(`Found ${files.length} files to index`);
+  const overallStart = Date.now();
   const results = [];
   for (const module of enabledModules) {
+    const moduleStart = Date.now();
     logger.info(`
 [${module.name}] Starting indexing...`);
     const moduleConfig = getModuleConfig(config, module.id);
@@ -4320,7 +4359,9 @@ async function indexDirectory(rootDir, options = {}) {
       };
       await module.initialize(configWithOverrides);
     }
-    const result = await indexWithModule(rootDir, files, module, config, verbose, introspection, logger, concurrency);
+    const moduleFiles = module.supportsFile ? files.filter((f) => module.supportsFile(f)) : files;
+    logger.info(`  Processing ${moduleFiles.length} files...`);
+    const result = await indexWithModule(rootDir, moduleFiles, module, config, verbose, introspection, logger, concurrency);
     results.push(result);
     if (module.finalize) {
       logger.info(`[${module.name}] Building secondary indexes...`);
@@ -4339,9 +4380,18 @@ async function indexDirectory(rootDir, options = {}) {
       };
       await module.finalize(ctx);
     }
-    logger.info(`[${module.name}] Complete: ${result.indexed} indexed, ${result.skipped} skipped, ${result.errors} errors`);
+    const moduleDuration = Date.now() - moduleStart;
+    result.durationMs = moduleDuration;
+    logger.info(`[${module.name}] Complete: ${result.indexed} indexed, ${result.skipped} skipped, ${result.errors} errors (${formatDuration(moduleDuration)})`);
   }
   await introspection.save(config);
+  const overallDuration = Date.now() - overallStart;
+  logger.info(`
+Indexing complete in ${formatDuration(overallDuration)}`);
+  const totalIndexed = results.reduce((sum, r) => sum + r.indexed, 0);
+  const totalSkipped = results.reduce((sum, r) => sum + r.skipped, 0);
+  const totalErrors = results.reduce((sum, r) => sum + r.errors, 0);
+  logger.info(`Total: ${totalIndexed} indexed, ${totalSkipped} skipped, ${totalErrors} errors`);
   await updateGlobalManifest(rootDir, enabledModules, config);
   return results;
 }
@@ -4805,13 +4855,14 @@ async function getIndexStatus(rootDir) {
   }
   return status;
 }
-var INDEX_SCHEMA_VERSION = "1.0.0", DEFAULT_CONCURRENCY = 4;
+var INDEX_SCHEMA_VERSION = "1.0.0", DEFAULT_CONCURRENCY;
 var init_indexer = __esm(() => {
   init_config2();
   init_registry();
   init_introspection2();
   init_logger();
   init_watcher();
+  DEFAULT_CONCURRENCY = getOptimalConcurrency();
 });
 
 // src/types.ts
@@ -4862,9 +4913,17 @@ async function search(rootDir, query, options = {}) {
     const moduleResults = await module.search(query, ctx, options);
     allResults.push(...moduleResults);
   }
-  allResults.sort((a, b) => b.score - a.score);
+  let filteredResults = allResults;
+  if (options.pathFilter && options.pathFilter.length > 0) {
+    const normalizedFilters = options.pathFilter.map((p) => p.replace(/\\/g, "/").replace(/^\//, "").replace(/\/$/, ""));
+    filteredResults = allResults.filter((result) => {
+      const normalizedPath = result.filepath.replace(/\\/g, "/");
+      return normalizedFilters.some((filter) => normalizedPath.startsWith(filter + "/") || normalizedPath === filter || normalizedPath.startsWith("./" + filter + "/") || normalizedPath === "./" + filter);
+    });
+  }
+  filteredResults.sort((a, b) => b.score - a.score);
   const topK = options.topK ?? 10;
-  return allResults.slice(0, topK);
+  return filteredResults.slice(0, topK);
 }
 function createSearchContext(rootDir, moduleId, config) {
   const indexPath = getModuleIndexPath(rootDir, moduleId, config);
@@ -4973,7 +5032,7 @@ init_logger();
 // package.json
 var package_default = {
   name: "raggrep",
-  version: "0.5.0",
+  version: "0.5.2",
   description: "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   type: "module",
   main: "./dist/index.js",
@@ -5117,6 +5176,17 @@ function parseFlags(args2) {
         console.error(`Invalid concurrency: ${args2[i]}. Must be a positive integer.`);
         process.exit(1);
       }
+    } else if (arg === "--filter" || arg === "-f") {
+      const filterPath = args2[++i];
+      if (filterPath) {
+        if (!flags.pathFilter) {
+          flags.pathFilter = [];
+        }
+        flags.pathFilter.push(filterPath);
+      } else {
+        console.error("--filter requires a path (e.g., src/auth)");
+        process.exit(1);
+      }
     } else if (!arg.startsWith("-")) {
       flags.remaining.push(arg);
     }
@@ -5138,7 +5208,7 @@ Usage:
 Options:
   -w, --watch              Watch for file changes and re-index automatically
   -m, --model <name>       Embedding model to use (default: all-MiniLM-L6-v2)
-  -c, --concurrency <n>    Number of files to process in parallel (default: 4)
+  -c, --concurrency <n>    Number of files to process in parallel (default: auto)
   -v, --verbose            Show detailed progress
   -h, --help               Show this help message
 
@@ -5225,6 +5295,7 @@ Options:
   -k, --top <n>        Number of results to return (default: 10)
   -s, --min-score <n>  Minimum similarity score 0-1 (default: 0.15)
   -t, --type <ext>     Filter by file extension (e.g., ts, tsx, js)
+  -f, --filter <path>  Filter by path prefix (can be used multiple times)
   -h, --help           Show this help message
 
 Note:
@@ -5239,6 +5310,8 @@ Examples:
   raggrep query "handle errors" --top 5
   raggrep query "database" --min-score 0.1
   raggrep query "interface" --type ts
+  raggrep query "login" --filter src/auth
+  raggrep query "api" --filter src/api --filter src/routes
 `);
         process.exit(0);
       }
@@ -5279,6 +5352,7 @@ Examples:
           topK: flags.topK ?? 10,
           minScore: flags.minScore,
           filePatterns,
+          pathFilter: flags.pathFilter,
           ensureFresh: false
         });
         console.log(formatSearchResults2(results));
@@ -5419,4 +5493,4 @@ Run 'raggrep <command> --help' for more information.
 }
 main();
 
-//# debugId=5CA623D9974ACF4364756E2164756E21
+//# debugId=3B601F4F22C997A464756E2164756E21