raggrep 0.2.2 → 0.3.0

package/dist/domain/entities/searchResult.d.ts

@@ -74,6 +74,17 @@ export interface SearchOptions {
     minScore?: number;
     /** Filter to specific file patterns (e.g., ['*.ts', '*.tsx']) */
     filePatterns?: string[];
+    /**
+     * Ensure the index is fresh before searching (default: true).
+     *
+     * When true, the search will automatically:
+     * - Create the index if it doesn't exist
+     * - Re-index any modified files
+     * - Remove entries for deleted files
+     *
+     * Set to false if you've already ensured freshness or want explicit control.
+     */
+    ensureFresh?: boolean;
 }
 /**
  * Default search options.

package/dist/index.js

@@ -10,7 +10,15 @@ var __export = (target, all) => {
 };
 var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
 // src/domain/entities/searchResult.ts
-var init_searchResult = () => {};
+var DEFAULT_SEARCH_OPTIONS;
+var init_searchResult = __esm(() => {
+  DEFAULT_SEARCH_OPTIONS = {
+    topK: 10,
+    minScore: 0.15,
+    filePatterns: [],
+    ensureFresh: true
+  };
+});
 
 // src/domain/entities/config.ts
 function createDefaultConfig() {
@@ -72,10 +80,17 @@ var init_config = __esm(() => {
     ".tsx",
     ".js",
     ".jsx",
+    ".mjs",
+    ".cjs",
     ".py",
     ".go",
     ".rs",
     ".java",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".sql",
     ".md",
     ".txt"
   ];
@@ -147,7 +162,7 @@ function getEmbeddingConfigFromModule(moduleConfig) {
   }
   return {
     model: modelName,
-    showProgress: options.showProgress !== false
+    showProgress: options.showProgress === true
   };
 }
 var DEFAULT_CONFIG, RAGGREP_TEMP_BASE, EMBEDDING_MODELS;
@@ -1847,7 +1862,6 @@ class CoreModule {
       bm25Data: this.bm25Index.serialize()
     };
     await fs2.writeFile(path5.join(coreDir, "symbols.json"), JSON.stringify(symbolIndexData, null, 2));
-    console.log(`  [Core] Symbol index built with ${this.symbolIndex.size} files`);
   }
   async search(query, ctx, options) {
     const config = ctx.config;
@@ -1970,7 +1984,10 @@ var init_core = __esm(() => {
 });
 
 // src/infrastructure/embeddings/transformersEmbedding.ts
-import { pipeline, env } from "@xenova/transformers";
+import {
+  pipeline,
+  env
+} from "@xenova/transformers";
 import * as path6 from "path";
 import * as os2 from "os";
 
@@ -1982,7 +1999,7 @@ class TransformersEmbeddingProvider {
   constructor(config) {
     this.config = {
       model: config?.model ?? "all-MiniLM-L6-v2",
-      showProgress: config?.showProgress ?? true
+      showProgress: config?.showProgress ?? false
     };
   }
   async initialize(config) {
@@ -2115,7 +2132,7 @@ var init_transformersEmbedding = __esm(() => {
   };
   globalConfig = {
     model: "all-MiniLM-L6-v2",
-    showProgress: true
+    showProgress: false
   };
 });
 
@@ -2687,6 +2704,57 @@ __export(exports_typescript, {
   DEFAULT_MIN_SCORE: () => DEFAULT_MIN_SCORE2
 });
 import * as path8 from "path";
+function detectQueryIntent(queryTerms) {
+  const hasImplementationTerm = queryTerms.some((term) => IMPLEMENTATION_TERMS.includes(term));
+  const hasDocumentationTerm = queryTerms.some((term) => DOCUMENTATION_TERMS.includes(term));
+  if (hasDocumentationTerm) {
+    return "documentation";
+  }
+  if (hasImplementationTerm) {
+    return "implementation";
+  }
+  return "neutral";
+}
+function calculateFileTypeBoost(filepath, queryTerms) {
+  const ext = path8.extname(filepath).toLowerCase();
+  const isSourceCode = SOURCE_CODE_EXTENSIONS.includes(ext);
+  const isDoc = DOC_EXTENSIONS.includes(ext);
+  const intent = detectQueryIntent(queryTerms);
+  if (intent === "implementation") {
+    if (isSourceCode) {
+      return 0.06;
+    }
+    return 0;
+  }
+  if (intent === "documentation") {
+    if (isDoc) {
+      return 0.08;
+    }
+    return 0;
+  }
+  return 0;
+}
+function calculateChunkTypeBoost(chunk) {
+  switch (chunk.type) {
+    case "function":
+      return 0.05;
+    case "class":
+    case "interface":
+      return 0.04;
+    case "type":
+    case "enum":
+      return 0.03;
+    case "variable":
+      return 0.02;
+    case "file":
+    case "block":
+    default:
+      return 0;
+  }
+}
+function calculateExportBoost(chunk) {
+  return chunk.isExported ? 0.03 : 0;
+}
 
 class TypeScriptModule {
   id = "language/typescript";
@@ -2732,7 +2800,9 @@ class TypeScriptModule {
       embeddings,
       embeddingModel: currentConfig.model
     };
-    const chunkTypes = [...new Set(parsedChunks.map((pc) => pc.type))];
+    const chunkTypes = [
+      ...new Set(parsedChunks.map((pc) => pc.type))
+    ];
     const exports = parsedChunks.filter((pc) => pc.isExported && pc.name).map((pc) => pc.name);
     const allKeywords = new Set;
     for (const pc of parsedChunks) {
@@ -2772,11 +2842,14 @@ class TypeScriptModule {
     }
     this.symbolicIndex.buildBM25Index();
     await this.symbolicIndex.save();
-    console.log(`  Symbolic index built with ${this.pendingSummaries.size} file summaries`);
     this.pendingSummaries.clear();
   }
   async search(query, ctx, options = {}) {
-    const { topK = DEFAULT_TOP_K2, minScore = DEFAULT_MIN_SCORE2, filePatterns } = options;
+    const {
+      topK = DEFAULT_TOP_K2,
+      minScore = DEFAULT_MIN_SCORE2,
+      filePatterns
+    } = options;
     const indexDir = getRaggrepDir(ctx.rootDir, ctx.config);
     const symbolicIndex = new SymbolicIndex(indexDir, this.id);
     let allFiles;
@@ -2851,7 +2924,11 @@ class TypeScriptModule {
       const semanticScore = cosineSimilarity(queryEmbedding, embedding);
       const bm25Score = bm25Scores.get(chunk.id) || 0;
       const pathBoost = pathBoosts.get(filepath) || 0;
-      const hybridScore = SEMANTIC_WEIGHT * semanticScore + BM25_WEIGHT * bm25Score + pathBoost;
+      const fileTypeBoost = calculateFileTypeBoost(filepath, queryTerms);
+      const chunkTypeBoost = calculateChunkTypeBoost(chunk);
+      const exportBoost = calculateExportBoost(chunk);
+      const totalBoost = pathBoost + fileTypeBoost + chunkTypeBoost + exportBoost;
+      const hybridScore = SEMANTIC_WEIGHT * semanticScore + BM25_WEIGHT * bm25Score + totalBoost;
       if (hybridScore >= minScore || bm25Score > 0.3) {
         results.push({
           filepath,
@@ -2861,7 +2938,10 @@ class TypeScriptModule {
           context: {
             semanticScore,
             bm25Score,
-            pathBoost
+            pathBoost,
+            fileTypeBoost,
+            chunkTypeBoost,
+            exportBoost
           }
         });
       }
@@ -2893,7 +2973,7 @@ class TypeScriptModule {
     return references;
   }
 }
-var DEFAULT_MIN_SCORE2 = 0.15, DEFAULT_TOP_K2 = 10, SEMANTIC_WEIGHT = 0.7, BM25_WEIGHT = 0.3;
+var DEFAULT_MIN_SCORE2 = 0.15, DEFAULT_TOP_K2 = 10, SEMANTIC_WEIGHT = 0.7, BM25_WEIGHT = 0.3, IMPLEMENTATION_TERMS, DOCUMENTATION_TERMS, SOURCE_CODE_EXTENSIONS, DOC_EXTENSIONS;
 var init_typescript = __esm(() => {
   init_embeddings();
   init_config2();
@@ -2901,6 +2981,52 @@ var init_typescript = __esm(() => {
   init_storage();
   init_keywords();
   init_keywords();
+  IMPLEMENTATION_TERMS = [
+    "function",
+    "method",
+    "class",
+    "interface",
+    "implement",
+    "implementation",
+    "endpoint",
+    "route",
+    "handler",
+    "controller",
+    "module",
+    "code"
+  ];
+  DOCUMENTATION_TERMS = [
+    "documentation",
+    "docs",
+    "guide",
+    "tutorial",
+    "readme",
+    "how",
+    "what",
+    "why",
+    "explain",
+    "overview",
+    "getting",
+    "started",
+    "requirements",
+    "setup",
+    "install",
+    "configure",
+    "configuration"
+  ];
+  SOURCE_CODE_EXTENSIONS = [
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".cjs",
+    ".py",
+    ".go",
+    ".rs",
+    ".java"
+  ];
+  DOC_EXTENSIONS = [".md", ".txt", ".rst"];
 });
 
 // src/app/indexer/index.ts
@@ -2913,10 +3039,9 @@ import * as path11 from "path";
 class ModuleRegistryImpl {
   modules = new Map;
   register(module) {
-    if (this.modules.has(module.id)) {
-      console.warn(`Module '${module.id}' is already registered, overwriting...`);
+    if (!this.modules.has(module.id)) {
+      this.modules.set(module.id, module);
     }
-    this.modules.set(module.id, module);
   }
   get(id) {
     return this.modules.get(id);
@@ -3155,7 +3280,6 @@ class IntrospectionIndex {
       await fs5.mkdir(path10.dirname(introFilePath), { recursive: true });
       await fs5.writeFile(introFilePath, JSON.stringify(intro, null, 2));
     }
-    console.log(`  [Introspection] Saved metadata for ${this.files.size} files`);
   }
   async load(config) {
     const introDir = path10.join(getRaggrepDir(this.rootDir, config), "introspection");
@@ -3276,6 +3400,163 @@ async function indexDirectory(rootDir, options = {}) {
   await updateGlobalManifest(rootDir, enabledModules, config);
   return results;
 }
+async function isIndexVersionCompatible(rootDir) {
+  const config = await loadConfig(rootDir);
+  const globalManifestPath = getGlobalManifestPath(rootDir, config);
+  try {
+    const content = await fs6.readFile(globalManifestPath, "utf-8");
+    const manifest = JSON.parse(content);
+    return manifest.version === INDEX_SCHEMA_VERSION;
+  } catch {
+    return false;
+  }
+}
+async function deleteIndex(rootDir) {
+  const indexDir = getRaggrepDir(rootDir);
+  try {
+    await fs6.rm(indexDir, { recursive: true, force: true });
+  } catch {}
+}
+async function ensureIndexFresh(rootDir, options = {}) {
+  const verbose = options.verbose ?? false;
+  const quiet = options.quiet ?? false;
+  rootDir = path11.resolve(rootDir);
+  const status = await getIndexStatus(rootDir);
+  if (!status.exists) {
+    if (!quiet) {
+      console.log(`No index found. Creating index...
+`);
+    }
+    const results = await indexDirectory(rootDir, { ...options, quiet });
+    const totalIndexed2 = results.reduce((sum, r) => sum + r.indexed, 0);
+    return { indexed: totalIndexed2, removed: 0, unchanged: 0 };
+  }
+  const versionCompatible = await isIndexVersionCompatible(rootDir);
+  if (!versionCompatible) {
+    if (!quiet) {
+      console.log(`Index version incompatible. Rebuilding...
+`);
+    }
+    await deleteIndex(rootDir);
+    const results = await indexDirectory(rootDir, { ...options, quiet });
+    const totalIndexed2 = results.reduce((sum, r) => sum + r.indexed, 0);
+    return { indexed: totalIndexed2, removed: 0, unchanged: 0 };
+  }
+  const config = await loadConfig(rootDir);
+  await registerBuiltInModules();
+  const enabledModules = registry.getEnabled(config);
+  if (enabledModules.length === 0) {
+    return { indexed: 0, removed: 0, unchanged: 0 };
+  }
+  const introspection = new IntrospectionIndex(rootDir);
+  await introspection.initialize();
+  const currentFiles = await findFiles(rootDir, config);
+  const currentFileSet = new Set(currentFiles.map((f) => path11.relative(rootDir, f)));
+  let totalIndexed = 0;
+  let totalRemoved = 0;
+  let totalUnchanged = 0;
+  for (const module of enabledModules) {
+    const moduleConfig = getModuleConfig(config, module.id);
+    if (module.initialize && moduleConfig) {
+      const configWithOverrides = { ...moduleConfig };
+      if (options.model && module.id === "language/typescript") {
+        configWithOverrides.options = {
+          ...configWithOverrides.options,
+          embeddingModel: options.model
+        };
+      }
+      await module.initialize(configWithOverrides);
+    }
+    const manifest = await loadModuleManifest(rootDir, module.id, config);
+    const indexPath = getModuleIndexPath(rootDir, module.id, config);
+    const filesToRemove = [];
+    for (const filepath of Object.keys(manifest.files)) {
+      if (!currentFileSet.has(filepath)) {
+        filesToRemove.push(filepath);
+      }
+    }
+    for (const filepath of filesToRemove) {
+      if (verbose) {
+        console.log(`  Removing stale: ${filepath}`);
+      }
+      const indexFilePath = path11.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
+      try {
+        await fs6.unlink(indexFilePath);
+      } catch {}
+      const symbolicFilePath = path11.join(indexPath, "symbolic", filepath.replace(/\.[^.]+$/, ".json"));
+      try {
+        await fs6.unlink(symbolicFilePath);
+      } catch {}
+      delete manifest.files[filepath];
+      totalRemoved++;
+    }
+    const ctx = {
+      rootDir,
+      config,
+      readFile: async (filepath) => {
+        const fullPath = path11.isAbsolute(filepath) ? filepath : path11.join(rootDir, filepath);
+        return fs6.readFile(fullPath, "utf-8");
+      },
+      getFileStats: async (filepath) => {
+        const fullPath = path11.isAbsolute(filepath) ? filepath : path11.join(rootDir, filepath);
+        const stats = await fs6.stat(fullPath);
+        return { lastModified: stats.mtime.toISOString() };
+      },
+      getIntrospection: (filepath) => introspection.getFile(filepath)
+    };
+    for (const filepath of currentFiles) {
+      const relativePath = path11.relative(rootDir, filepath);
+      try {
+        const stats = await fs6.stat(filepath);
+        const lastModified = stats.mtime.toISOString();
+        const existingEntry = manifest.files[relativePath];
+        if (existingEntry && existingEntry.lastModified === lastModified) {
+          totalUnchanged++;
+          continue;
+        }
+        if (verbose) {
+          console.log(`  Indexing: ${relativePath}`);
+        }
+        const content = await fs6.readFile(filepath, "utf-8");
+        introspection.addFile(relativePath, content);
+        const fileIndex = await module.indexFile(relativePath, content, ctx);
+        if (fileIndex) {
+          await writeFileIndex(rootDir, module.id, relativePath, fileIndex, config);
+          manifest.files[relativePath] = {
+            lastModified,
+            chunkCount: fileIndex.chunks.length
+          };
+          totalIndexed++;
+        }
+      } catch (error) {
+        if (verbose) {
+          console.error(`  Error indexing ${relativePath}:`, error);
+        }
+      }
+    }
+    if (totalIndexed > 0 || totalRemoved > 0) {
+      manifest.lastUpdated = new Date().toISOString();
+      await writeModuleManifest(rootDir, module.id, manifest, config);
+      if (module.finalize) {
+        await module.finalize(ctx);
+      }
+    }
+    if (totalRemoved > 0) {
+      await cleanupEmptyDirectories(indexPath);
+    }
+  }
+  if (totalIndexed > 0) {
+    await introspection.save(config);
+  }
+  if (totalIndexed > 0 || totalRemoved > 0) {
+    await updateGlobalManifest(rootDir, enabledModules, config);
+  }
+  return {
+    indexed: totalIndexed,
+    removed: totalRemoved,
+    unchanged: totalUnchanged
+  };
+}
 async function indexWithModule(rootDir, files, module, config, verbose, introspection) {
   const result = {
     moduleId: module.id,
@@ -3464,19 +3745,85 @@ async function cleanupEmptyDirectories(dir) {
     return false;
   }
 }
+async function getIndexStatus(rootDir) {
+  rootDir = path11.resolve(rootDir);
+  const config = await loadConfig(rootDir);
+  const location = getIndexLocation(rootDir);
+  const indexDir = location.indexDir;
+  const status = {
+    exists: false,
+    rootDir,
+    indexDir,
+    modules: [],
+    totalFiles: 0
+  };
+  try {
+    await fs6.access(indexDir);
+  } catch {
+    return status;
+  }
+  try {
+    const globalManifestPath = getGlobalManifestPath(rootDir, config);
+    const content = await fs6.readFile(globalManifestPath, "utf-8");
+    const globalManifest = JSON.parse(content);
+    status.exists = true;
+    status.lastUpdated = globalManifest.lastUpdated;
+    for (const moduleId of globalManifest.modules) {
+      try {
+        const manifest = await loadModuleManifest(rootDir, moduleId, config);
+        const fileCount = Object.keys(manifest.files).length;
+        status.modules.push({
+          id: moduleId,
+          fileCount,
+          lastUpdated: manifest.lastUpdated
+        });
+        status.totalFiles += fileCount;
+      } catch {}
+    }
+  } catch {
+    try {
+      const entries = await fs6.readdir(path11.join(indexDir, "index"));
+      if (entries.length > 0) {
+        status.exists = true;
+        for (const entry of entries) {
+          try {
+            const manifest = await loadModuleManifest(rootDir, entry, config);
+            const fileCount = Object.keys(manifest.files).length;
+            status.modules.push({
+              id: entry,
+              fileCount,
+              lastUpdated: manifest.lastUpdated
+            });
+            status.totalFiles += fileCount;
+          } catch {}
+        }
+      }
+    } catch {}
+  }
+  return status;
+}
 
 // src/app/search/index.ts
-init_config2();
 import * as fs7 from "fs/promises";
 import * as path12 from "path";
+
+// src/types.ts
+init_entities();
+
+// src/app/search/index.ts
+init_config2();
 async function search(rootDir, query, options = {}) {
   rootDir = path12.resolve(rootDir);
+  const ensureFresh = options.ensureFresh ?? DEFAULT_SEARCH_OPTIONS.ensureFresh;
+  if (ensureFresh) {
+    await ensureIndexFresh(rootDir, { quiet: true });
+  }
   console.log(`Searching for: "${query}"`);
   const config = await loadConfig(rootDir);
   await registerBuiltInModules();
   const globalManifest = await loadGlobalManifest(rootDir, config);
   if (!globalManifest || globalManifest.modules.length === 0) {
-    console.log('No index found. Run "bun run index" first.');
+    console.log('No index found. Run "raggrep index" first.');
     return [];
   }
   const modulesToSearch = [];
@@ -3624,4 +3971,4 @@ export {
   cleanup
 };
 
-//# debugId=719044F94BD570BC64756E2164756E21
+//# debugId=25853E0D892AD2D964756E2164756E21