raggrep 0.7.0 → 0.8.0

package/dist/index.js

@@ -133,11 +133,28 @@ var init_config = __esm(() => {
 // src/domain/entities/literal.ts
 var init_literal = () => {};
 
+// src/domain/entities/lexicon.ts
+var DEFAULT_EXPANSION_OPTIONS, EXPANSION_WEIGHTS;
+var init_lexicon = __esm(() => {
+  DEFAULT_EXPANSION_OPTIONS = {
+    maxDepth: 1,
+    includeWeak: true,
+    maxTerms: 20,
+    minTermLength: 2
+  };
+  EXPANSION_WEIGHTS = {
+    strong: 0.9,
+    moderate: 0.6,
+    weak: 0.3
+  };
+});
+
 // src/domain/entities/index.ts
 var init_entities = __esm(() => {
   init_searchResult();
   init_config();
   init_literal();
+  init_lexicon();
 });
 
 // src/infrastructure/config/configLoader.ts
@@ -2856,6 +2873,586 @@ var init_literalScorer = __esm(() => {
   };
 });
 
+// src/domain/services/lexicon.ts
+function buildLookupMap(lexicon) {
+  const map = new Map;
+  for (const entry of lexicon.entries) {
+    map.set(entry.term.toLowerCase(), entry);
+  }
+  return map;
+}
+function getSynonyms(term, lexicon = DEFAULT_LEXICON) {
+  const lookupMap = lexicon === DEFAULT_LEXICON ? defaultLookupMap : buildLookupMap(lexicon);
+  const entry = lookupMap.get(term.toLowerCase());
+  return entry ? entry.synonyms : [];
+}
+function tokenizeQuery(query) {
+  const stopWords = new Set([
+    "the",
+    "a",
+    "an",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "could",
+    "should",
+    "may",
+    "might",
+    "must",
+    "shall",
+    "can",
+    "need",
+    "dare",
+    "ought",
+    "used",
+    "to",
+    "of",
+    "in",
+    "for",
+    "on",
+    "with",
+    "at",
+    "by",
+    "from",
+    "as",
+    "into",
+    "through",
+    "during",
+    "before",
+    "after",
+    "above",
+    "below",
+    "between",
+    "under",
+    "again",
+    "further",
+    "then",
+    "once",
+    "here",
+    "there",
+    "when",
+    "where",
+    "why",
+    "how",
+    "all",
+    "each",
+    "few",
+    "more",
+    "most",
+    "other",
+    "some",
+    "such",
+    "no",
+    "nor",
+    "not",
+    "only",
+    "own",
+    "same",
+    "so",
+    "than",
+    "too",
+    "very",
+    "just",
+    "and",
+    "but",
+    "if",
+    "or",
+    "because",
+    "until",
+    "while",
+    "this",
+    "that",
+    "these",
+    "those",
+    "what",
+    "which",
+    "who",
+    "whom",
+    "i",
+    "me",
+    "my",
+    "we",
+    "our",
+    "you",
+    "your",
+    "he",
+    "him",
+    "his",
+    "she",
+    "her",
+    "it",
+    "its",
+    "they",
+    "them",
+    "their"
+  ]);
+  return query.toLowerCase().split(/\s+/).filter((term) => term.length > 0 && !stopWords.has(term));
+}
+function expandQuery(query, lexicon = DEFAULT_LEXICON, options = {}) {
+  const opts = { ...DEFAULT_EXPANSION_OPTIONS, ...options };
+  const originalTerms = tokenizeQuery(query);
+  const expandedTerms = [];
+  const seenTerms = new Set;
+  for (const term of originalTerms) {
+    if (term.length >= opts.minTermLength && !seenTerms.has(term)) {
+      expandedTerms.push({
+        term,
+        weight: 1,
+        source: "original"
+      });
+      seenTerms.add(term);
+    }
+  }
+  if (opts.maxDepth >= 1) {
+    for (const term of originalTerms) {
+      if (term.length < opts.minTermLength)
+        continue;
+      const synonyms = getSynonyms(term, lexicon);
+      for (const syn of synonyms) {
+        if (syn.grade === "weak" && !opts.includeWeak)
+          continue;
+        const synLower = syn.term.toLowerCase();
+        if (seenTerms.has(synLower))
+          continue;
+        if (expandedTerms.length >= opts.maxTerms)
+          break;
+        expandedTerms.push({
+          term: syn.term,
+          weight: EXPANSION_WEIGHTS[syn.grade],
+          source: syn.grade,
+          expandedFrom: term
+        });
+        seenTerms.add(synLower);
+      }
+      if (expandedTerms.length >= opts.maxTerms)
+        break;
+    }
+  }
+  const originalPart = originalTerms.join(" ");
+  const synonymPart = expandedTerms.filter((t) => t.source !== "original").map((t) => t.term).join(" ");
+  const expandedQueryString = synonymPart ? `${originalPart} ${synonymPart}` : originalPart;
+  return {
+    originalQuery: query,
+    originalTerms,
+    expandedTerms,
+    expandedQueryString,
+    wasExpanded: expandedTerms.some((t) => t.source !== "original")
+  };
+}
+var DEFAULT_LEXICON, defaultLookupMap;
+var init_lexicon2 = __esm(() => {
+  init_lexicon();
+  DEFAULT_LEXICON = {
+    version: "1.0.0",
+    entries: [
+      {
+        term: "function",
+        synonyms: [
+          { term: "method", grade: "strong" },
+          { term: "func", grade: "strong" },
+          { term: "handler", grade: "moderate" },
+          { term: "callback", grade: "moderate" },
+          { term: "procedure", grade: "weak" },
+          { term: "routine", grade: "weak" }
+        ]
+      },
+      {
+        term: "method",
+        synonyms: [
+          { term: "function", grade: "strong" },
+          { term: "func", grade: "strong" },
+          { term: "handler", grade: "moderate" }
+        ]
+      },
+      {
+        term: "class",
+        synonyms: [
+          { term: "type", grade: "moderate" },
+          { term: "interface", grade: "moderate" },
+          { term: "struct", grade: "moderate" },
+          { term: "model", grade: "weak" },
+          { term: "entity", grade: "weak" }
+        ]
+      },
+      {
+        term: "interface",
+        synonyms: [
+          { term: "type", grade: "strong" },
+          { term: "contract", grade: "moderate" },
+          { term: "protocol", grade: "weak" }
+        ]
+      },
+      {
+        term: "type",
+        synonyms: [
+          { term: "interface", grade: "strong" },
+          { term: "typedef", grade: "strong" },
+          { term: "schema", grade: "moderate" }
+        ]
+      },
+      {
+        term: "variable",
+        synonyms: [
+          { term: "var", grade: "strong" },
+          { term: "const", grade: "strong" },
+          { term: "constant", grade: "strong" },
+          { term: "property", grade: "moderate" },
+          { term: "field", grade: "moderate" }
+        ]
+      },
+      {
+        term: "constant",
+        synonyms: [
+          { term: "const", grade: "strong" },
+          { term: "variable", grade: "moderate" },
+          { term: "config", grade: "weak" }
+        ]
+      },
+      {
+        term: "auth",
+        synonyms: [
+          { term: "authentication", grade: "strong" },
+          { term: "authorization", grade: "strong" },
+          { term: "login", grade: "moderate" },
+          { term: "signin", grade: "moderate" },
+          { term: "session", grade: "weak" },
+          { term: "security", grade: "weak" }
+        ]
+      },
+      {
+        term: "authentication",
+        synonyms: [
+          { term: "auth", grade: "strong" },
+          { term: "login", grade: "moderate" },
+          { term: "signin", grade: "moderate" },
+          { term: "identity", grade: "weak" }
+        ]
+      },
+      {
+        term: "authorization",
+        synonyms: [
+          { term: "auth", grade: "strong" },
+          { term: "permission", grade: "moderate" },
+          { term: "access", grade: "moderate" },
+          { term: "role", grade: "weak" }
+        ]
+      },
+      {
+        term: "login",
+        synonyms: [
+          { term: "signin", grade: "strong" },
+          { term: "auth", grade: "moderate" },
+          { term: "authenticate", grade: "moderate" }
+        ]
+      },
+      {
+        term: "logout",
+        synonyms: [
+          { term: "signout", grade: "strong" },
+          { term: "logoff", grade: "strong" }
+        ]
+      },
+      {
+        term: "password",
+        synonyms: [
+          { term: "pwd", grade: "strong" },
+          { term: "pass", grade: "strong" },
+          { term: "credential", grade: "moderate" },
+          { term: "secret", grade: "weak" }
+        ]
+      },
+      {
+        term: "token",
+        synonyms: [
+          { term: "jwt", grade: "strong" },
+          { term: "bearer", grade: "moderate" },
+          { term: "credential", grade: "weak" }
+        ]
+      },
+      {
+        term: "database",
+        synonyms: [
+          { term: "db", grade: "strong" },
+          { term: "datastore", grade: "strong" },
+          { term: "storage", grade: "moderate" },
+          { term: "repository", grade: "weak" }
+        ]
+      },
+      {
+        term: "query",
+        synonyms: [
+          { term: "select", grade: "moderate" },
+          { term: "find", grade: "moderate" },
+          { term: "fetch", grade: "moderate" },
+          { term: "search", grade: "weak" }
+        ]
+      },
+      {
+        term: "insert",
+        synonyms: [
+          { term: "create", grade: "strong" },
+          { term: "add", grade: "strong" },
+          { term: "save", grade: "moderate" },
+          { term: "store", grade: "moderate" }
+        ]
+      },
+      {
+        term: "update",
+        synonyms: [
+          { term: "modify", grade: "strong" },
+          { term: "edit", grade: "strong" },
+          { term: "patch", grade: "moderate" },
+          { term: "change", grade: "moderate" }
+        ]
+      },
+      {
+        term: "delete",
+        synonyms: [
+          { term: "remove", grade: "strong" },
+          { term: "destroy", grade: "strong" },
+          { term: "drop", grade: "moderate" },
+          { term: "erase", grade: "weak" }
+        ]
+      },
+      {
+        term: "cache",
+        synonyms: [
+          { term: "redis", grade: "moderate" },
+          { term: "memcache", grade: "moderate" },
+          { term: "store", grade: "weak" },
+          { term: "buffer", grade: "weak" }
+        ]
+      },
+      {
+        term: "api",
+        synonyms: [
+          { term: "endpoint", grade: "strong" },
+          { term: "route", grade: "moderate" },
+          { term: "rest", grade: "moderate" },
+          { term: "service", grade: "weak" }
+        ]
+      },
+      {
+        term: "endpoint",
+        synonyms: [
+          { term: "api", grade: "strong" },
+          { term: "route", grade: "strong" },
+          { term: "path", grade: "moderate" }
+        ]
+      },
+      {
+        term: "request",
+        synonyms: [
+          { term: "req", grade: "strong" },
+          { term: "call", grade: "moderate" },
+          { term: "fetch", grade: "moderate" }
+        ]
+      },
+      {
+        term: "response",
+        synonyms: [
+          { term: "res", grade: "strong" },
+          { term: "reply", grade: "moderate" },
+          { term: "result", grade: "weak" }
+        ]
+      },
+      {
+        term: "middleware",
+        synonyms: [
+          { term: "interceptor", grade: "moderate" },
+          { term: "filter", grade: "moderate" },
+          { term: "handler", grade: "weak" }
+        ]
+      },
+      {
+        term: "error",
+        synonyms: [
+          { term: "exception", grade: "strong" },
+          { term: "err", grade: "strong" },
+          { term: "failure", grade: "moderate" },
+          { term: "fault", grade: "weak" }
+        ]
+      },
+      {
+        term: "exception",
+        synonyms: [
+          { term: "error", grade: "strong" },
+          { term: "throw", grade: "moderate" },
+          { term: "catch", grade: "moderate" }
+        ]
+      },
+      {
+        term: "validate",
+        synonyms: [
+          { term: "verify", grade: "strong" },
+          { term: "check", grade: "strong" },
+          { term: "assert", grade: "moderate" },
+          { term: "ensure", grade: "moderate" }
+        ]
+      },
+      {
+        term: "config",
+        synonyms: [
+          { term: "configuration", grade: "strong" },
+          { term: "settings", grade: "strong" },
+          { term: "options", grade: "moderate" },
+          { term: "env", grade: "weak" },
+          { term: "environment", grade: "weak" }
+        ]
+      },
+      {
+        term: "environment",
+        synonyms: [
+          { term: "env", grade: "strong" },
+          { term: "config", grade: "moderate" },
+          { term: "settings", grade: "weak" }
+        ]
+      },
+      {
+        term: "test",
+        synonyms: [
+          { term: "spec", grade: "strong" },
+          { term: "unittest", grade: "strong" },
+          { term: "check", grade: "moderate" },
+          { term: "verify", grade: "weak" }
+        ]
+      },
+      {
+        term: "mock",
+        synonyms: [
+          { term: "stub", grade: "strong" },
+          { term: "fake", grade: "strong" },
+          { term: "spy", grade: "moderate" },
+          { term: "double", grade: "weak" }
+        ]
+      },
+      {
+        term: "async",
+        synonyms: [
+          { term: "asynchronous", grade: "strong" },
+          { term: "await", grade: "moderate" },
+          { term: "promise", grade: "moderate" }
+        ]
+      },
+      {
+        term: "callback",
+        synonyms: [
+          { term: "handler", grade: "strong" },
+          { term: "listener", grade: "moderate" },
+          { term: "hook", grade: "moderate" }
+        ]
+      },
+      {
+        term: "event",
+        synonyms: [
+          { term: "emit", grade: "moderate" },
+          { term: "trigger", grade: "moderate" },
+          { term: "signal", grade: "weak" },
+          { term: "message", grade: "weak" }
+        ]
+      },
+      {
+        term: "util",
+        synonyms: [
+          { term: "utility", grade: "strong" },
+          { term: "utils", grade: "strong" },
+          { term: "helper", grade: "strong" },
+          { term: "common", grade: "weak" }
+        ]
+      },
+      {
+        term: "helper",
+        synonyms: [
+          { term: "util", grade: "strong" },
+          { term: "utility", grade: "strong" },
+          { term: "support", grade: "weak" }
+        ]
+      },
+      {
+        term: "parse",
+        synonyms: [
+          { term: "decode", grade: "moderate" },
+          { term: "deserialize", grade: "moderate" },
+          { term: "extract", grade: "weak" }
+        ]
+      },
+      {
+        term: "serialize",
+        synonyms: [
+          { term: "encode", grade: "moderate" },
+          { term: "stringify", grade: "moderate" },
+          { term: "convert", grade: "weak" }
+        ]
+      },
+      {
+        term: "get",
+        synonyms: [
+          { term: "fetch", grade: "strong" },
+          { term: "retrieve", grade: "strong" },
+          { term: "find", grade: "moderate" },
+          { term: "load", grade: "moderate" }
+        ]
+      },
+      {
+        term: "set",
+        synonyms: [
+          { term: "assign", grade: "strong" },
+          { term: "store", grade: "moderate" },
+          { term: "save", grade: "moderate" }
+        ]
+      },
+      {
+        term: "find",
+        synonyms: [
+          { term: "search", grade: "strong" },
+          { term: "locate", grade: "strong" },
+          { term: "lookup", grade: "moderate" },
+          { term: "get", grade: "moderate" }
+        ]
+      },
+      {
+        term: "create",
+        synonyms: [
+          { term: "make", grade: "strong" },
+          { term: "build", grade: "strong" },
+          { term: "new", grade: "moderate" },
+          { term: "generate", grade: "moderate" }
+        ]
+      },
+      {
+        term: "send",
+        synonyms: [
+          { term: "emit", grade: "moderate" },
+          { term: "dispatch", grade: "moderate" },
+          { term: "post", grade: "moderate" },
+          { term: "transmit", grade: "weak" }
+        ]
+      },
+      {
+        term: "receive",
+        synonyms: [
+          { term: "accept", grade: "moderate" },
+          { term: "handle", grade: "moderate" },
+          { term: "process", grade: "weak" }
+        ]
+      }
+    ]
+  };
+  defaultLookupMap = buildLookupMap(DEFAULT_LEXICON);
+});
+
 // src/domain/services/index.ts
 var init_services = __esm(() => {
   init_keywords();
@@ -2863,6 +3460,7 @@ var init_services = __esm(() => {
   init_queryLiteralParser();
   init_literalExtractor();
   init_literalScorer();
+  init_lexicon2();
 });
 
 // src/modules/language/typescript/parseCode.ts
@@ -3183,6 +3781,11 @@ var init_symbolicIndex = __esm(() => {
 });
 
 // src/infrastructure/storage/literalIndex.ts
+var exports_literalIndex = {};
+__export(exports_literalIndex, {
+  getLiteralIndexPath: () => getLiteralIndexPath,
+  LiteralIndex: () => LiteralIndex
+});
 import * as fs4 from "fs/promises";
 import * as path9 from "path";
 
@@ -3235,6 +3838,22 @@ class LiteralIndex {
       }
     }
   }
+  removeFile(filepath) {
+    let removed = 0;
+    for (const [key, entries] of this.entries) {
+      const filtered = entries.filter((e) => e.filepath !== filepath);
+      const removedCount = entries.length - filtered.length;
+      if (removedCount > 0) {
+        removed += removedCount;
+        if (filtered.length === 0) {
+          this.entries.delete(key);
+        } else {
+          this.entries.set(key, filtered);
+        }
+      }
+    }
+    return removed;
+  }
   findMatches(queryLiterals) {
     const matches = [];
     for (const queryLiteral of queryLiterals) {
@@ -3327,6 +3946,9 @@ function shouldReplaceMatchType(existing, incoming) {
   };
   return priority[incoming] > priority[existing];
 }
+function getLiteralIndexPath(rootDir, moduleId, indexDir = ".raggrep") {
+  return path9.join(rootDir, indexDir, "index", moduleId, "literals");
+}
 var init_literalIndex = () => {};
 
 // src/infrastructure/storage/index.ts
@@ -3489,6 +4111,16 @@ class TypeScriptModule {
     await this.symbolicIndex.save();
     this.literalIndex = new LiteralIndex(indexDir, this.id);
     await this.literalIndex.initialize();
+    const indexedFilepaths = new Set;
+    for (const filepath of this.pendingSummaries.keys()) {
+      indexedFilepaths.add(filepath);
+    }
+    for (const { filepath } of this.pendingLiterals.values()) {
+      indexedFilepaths.add(filepath);
+    }
+    for (const filepath of indexedFilepaths) {
+      this.literalIndex.removeFile(filepath);
+    }
     for (const [chunkId, { filepath, literals }] of this.pendingLiterals) {
       this.literalIndex.addLiterals(chunkId, filepath, literals);
     }
@@ -3531,7 +4163,12 @@ class TypeScriptModule {
       });
     }
     const semanticQuery = remainingQuery.trim() || query;
-    const queryEmbedding = await getEmbedding(semanticQuery);
+    const expandedQuery = expandQuery(semanticQuery, undefined, {
+      maxDepth: 1,
+      includeWeak: false,
+      maxTerms: 10
+    });
+    const queryEmbedding = await getEmbedding(expandedQuery.expandedQueryString);
     const bm25Index = new BM25Index;
     const allChunksData = [];
     for (const filepath of filesToSearch) {
@@ -3611,7 +4248,8 @@ class TypeScriptModule {
             literalMultiplier: literalContribution.multiplier,
             literalMatchType: literalContribution.bestMatchType,
             literalConfidence: literalContribution.bestConfidence,
-            literalMatchCount: literalContribution.matchCount
+            literalMatchCount: literalContribution.matchCount,
+            synonymsUsed: expandedQuery.wasExpanded ? expandedQuery.expandedTerms.filter((t) => t.source !== "original").map((t) => t.term) : undefined
           }
         });
       }
@@ -4846,6 +5484,7 @@ async function ensureIndexFresh(rootDir, options = {}) {
         filesToRemove.push(filepath);
       }
     }
+    const removedFilepaths = [];
     for (const filepath of filesToRemove) {
       logger.debug(`  Removing stale: ${filepath}`);
       const indexFilePath = path15.join(indexPath, filepath.replace(/\.[^.]+$/, ".json"));
@@ -4857,8 +5496,21 @@ async function ensureIndexFresh(rootDir, options = {}) {
         await fs7.unlink(symbolicFilePath);
       } catch {}
       delete manifest.files[filepath];
+      removedFilepaths.push(filepath);
       totalRemoved++;
     }
+    if (removedFilepaths.length > 0) {
+      try {
+        const { LiteralIndex: LiteralIndex2 } = await Promise.resolve().then(() => (init_literalIndex(), exports_literalIndex));
+        const raggrepDir = getRaggrepDir(rootDir, config);
+        const literalIndex = new LiteralIndex2(raggrepDir, module.id);
+        await literalIndex.initialize();
+        for (const filepath of removedFilepaths) {
+          literalIndex.removeFile(filepath);
+        }
+        await literalIndex.save();
+      } catch {}
+    }
     const ctx = {
       rootDir,
       config,
@@ -5406,4 +6058,4 @@ export {
   ConsoleLogger
 };
 
-//# debugId=8F3468BC25F1B30164756E2164756E21
+//# debugId=59B4DA12592C31BA64756E2164756E21