scai 0.1.163 → 0.1.164

package/dist/db/fileIndex.js

@@ -100,54 +100,143 @@ export function queryFiles(safeQuery, limit = 10) {
 // - Uses LLM aggressively
 // - Optimizes for precision
 // --------------------------------------------------
-export async function semanticSearchFiles(originalQuery, query, topK = 5) {
+export async function semanticSearchFiles(originalQuery, _query, // ignored now – LLM owns query construction
+topK = 5) {
     const db = getDbForRepo();
-    const seen = new Map();
-    // -----------------------------
-    // Primary FTS
-    // -----------------------------
-    const safeQuery = sanitizeQueryForFts(query);
+    // --------------------------------------------------
+    // 1. LLM → primary FTS query (always)
+    // --------------------------------------------------
+    const primaryFtsQuery = await generatePrimaryFtsQuery(originalQuery);
+    logInputOutput("semanticSearchFiles LLM primary query", "output", {
+        originalQuery,
+        ftsQuery: primaryFtsQuery,
+    });
+    // --------------------------------------------------
+    // 2. Run primary FTS once
+    // --------------------------------------------------
     const primaryResults = db
         .prepare(sqlTemplates.searchFilesTemplate)
-        .all(safeQuery, RELATED_FILES_LIMIT);
-    logInputOutput("semanticSearchFiles primary FTS", "input", {
-        safeQuery,
-        count: primaryResults.length,
-    });
-    if (primaryResults.length === 0) {
-        // No FTS at all → blind expansion
-        return semanticFallbackSearch(originalQuery, topK);
+        .all(primaryFtsQuery, RELATED_FILES_LIMIT);
+    if (primaryResults.length > 0) {
+        return rankAndMap(new Map(primaryResults.map(r => [r.id, r])), topK);
     }
-    // -----------------------------
-    // Relevance check (KEY STEP)
-    // -----------------------------
-    const relevance = await checkFtsRelevanceWithModel(originalQuery, primaryResults);
-    logInputOutput("semanticSearchFiles relevance check", "output", relevance);
-    // -----------------------------
-    // If relevant → trust FTS
-    // -----------------------------
-    if (relevance.relevant) {
-        primaryResults.forEach(r => seen.set(r.id, r));
-        return rankAndMap(seen, topK);
-    }
-    // -----------------------------
-    // Not relevant → model-guided expansion
-    // -----------------------------
-    for (const term of relevance.suggestedTerms) {
-        const safeTerm = sanitizeQueryForFts(term);
+    // --------------------------------------------------
+    // 3. Fallback: LLM → 2–3 subqueries (ONLY if zero results)
+    // --------------------------------------------------
+    const subQueries = await generateFallbackFtsQueries(originalQuery, primaryFtsQuery);
+    logInputOutput("semanticSearchFiles LLM fallback queries", "output", {
+        originalQuery,
+        primaryFtsQuery,
+        subQueries,
+    });
+    // --------------------------------------------------
+    // 4. Execute fallback queries sequentially
+    // --------------------------------------------------
+    for (const subQuery of subQueries) {
         const rows = db
             .prepare(sqlTemplates.searchFilesTemplate)
-            .all(safeTerm, RELATED_FILES_LIMIT);
-        for (const row of rows) {
-            if (!seen.has(row.id))
-                seen.set(row.id, row);
+            .all(subQuery, RELATED_FILES_LIMIT);
+        if (rows.length > 0) {
+            return rankAndMap(new Map(rows.map(r => [r.id, r])), topK);
+        }
+    }
+    // --------------------------------------------------
+    // 5. Hard stop
+    // --------------------------------------------------
+    return [];
+}
+async function generatePrimaryFtsQuery(userQuery) {
+    const prompt = `
+You are generating a SQLite FTS query for searching a source code repository.
+
+Input (natural language):
+"${userQuery}"
+
+Task:
+- Produce ONE concise FTS query
+- Focus on filenames, symbols, module names, domain nouns
+- Prefer literal identifiers likely to exist in code
+- NO sentences
+- NO stopwords
+- NO explanations
+- NO wildcards unless absolutely necessary
+- Use OR between terms
+- **MAX 10 terms only** — be selective and concise
+
+Output JSON ONLY:
+{
+  "ftsQuery": "term1 OR term2 OR term3"
+}
+`.trim();
+    try {
+        const response = await generate({ content: prompt, query: "" });
+        const cleaned = await cleanupModule.run({
+            query: userQuery,
+            content: response.data,
+        });
+        if (cleaned.data &&
+            typeof cleaned.data === "object" &&
+            "ftsQuery" in cleaned.data &&
+            typeof cleaned.data.ftsQuery === "string") {
+            return cleaned.data.ftsQuery;
         }
     }
-    // Safety net: never return empty if we had FTS
-    if (seen.size === 0) {
-        primaryResults.forEach(r => seen.set(r.id, r));
+    catch (err) {
+        log(`⚠️ [semanticSearchFiles] Failed to generate primary FTS query: ${String(err)}`);
     }
-    return rankAndMap(seen, topK);
+    // Absolute safety fallback — never explode
+    return sanitizeQueryForFts(userQuery);
+}
+async function generateFallbackFtsQueries(userQuery, failedQuery) {
+    const prompt = `
+You are generating fallback SQLite FTS queries for a source code search.
+
+Original user query:
+"${userQuery}"
+
+Primary FTS query returned ZERO results:
+"${failedQuery}"
+
+Task:
+- Generate 2–3 independent FTS queries (MAX 3)
+- Each query should be concise: no more than 10 OR-joined search terms
+- Focus on filenames, symbols, module names
+- Avoid natural-language sentences
+- Avoid recursion or refinement loops
+- Use OR between terms
+
+Output JSON ONLY:
+{
+  "subQueries": [
+    "query1",
+    "query2",
+    "query3"
+  ]
+}
+`.trim();
+    try {
+        const response = await generate({ content: prompt, query: "" });
+        const cleaned = await cleanupModule.run({
+            query: userQuery,
+            content: response.data,
+        });
+        if (cleaned.data &&
+            typeof cleaned.data === "object" &&
+            Array.isArray(cleaned.data.subQueries)) {
+            return cleaned.data.subQueries
+                .filter((q) => typeof q === "string")
+                .slice(0, 3) // cap to 3 queries
+                .map((q) => q
+                .split(' OR ')
+                .map(term => sanitizeQueryForFts(term)) // sanitize each term individually
+                .slice(0, 10) // cap terms per query
+                .join(' OR '));
+        }
+    }
+    catch (err) {
+        log(`⚠️ [semanticSearchFiles] Failed to generate fallback queries: ${String(err)}`);
+    }
+    return [];
 }
 // --------------------------------------------------
 // PLANNER SEARCH (fileSearchModule, discovery)
@@ -209,71 +298,6 @@ function rankAndMap(seen, topK) {
         bm25Score: r.bm25Score,
     }));
 }
-/**
- * Ask the model whether the initial FTS hits are relevant to the query.
- * If not, suggest new search terms for another FTS pass.
- */
-async function checkFtsRelevanceWithModel(query, ftsResults) {
-    if (ftsResults.length === 0) {
-        return { relevant: false, suggestedTerms: [] };
-    }
-    const prompt = `
-You are assisting a code search system that uses full-text search (FTS)
-over source code.
-
-Query (natural language):
-"${query}"
-
-Initial FTS results (filenames and summaries):
-${JSON.stringify(ftsResults)}
-
-Task:
-1. Decide whether these results are relevant to the query.
-2. If they are NOT relevant, suggest alternative search terms.
-
-IMPORTANT RULES FOR SUGGESTED TERMS:
-- Terms MUST be likely to appear literally in source code.
-- Prefer: filenames, module names, function names, variables, symbols, config keys.
-- Use short identifiers (1–3 words max).
-- Avoid natural-language phrases or explanations.
-- Avoid conceptual or architectural descriptions.
-- Examples of GOOD terms: "api", "router", "frontend", "backend", "client", "server", "routes", "config.ts"
-- Examples of BAD terms: "frontend backend separation", "code architecture", "business logic"
-
-Output format:
-- If relevant:
-  { "relevant": true, "suggestedTerms": [] }
-- If not relevant:
-  { "relevant": false, "suggestedTerms": ["term1", "term2", "term3"] }
-
-Return ONLY valid JSON.
-`.trim();
-    try {
-        const response = await generate({ content: prompt, query: "" });
-        const cleaned = await cleanupModule.run({
-            query,
-            content: response.data,
-        });
-        const data = cleaned.data;
-        // Type guard: ensure it's an object with correct properties
-        if (data &&
-            typeof data === "object" &&
-            "relevant" in data &&
-            "suggestedTerms" in data &&
-            typeof data.relevant === "boolean" &&
-            Array.isArray(data.suggestedTerms)) {
-            const relevant = data.relevant;
-            const suggestedTerms = data.suggestedTerms.filter((t) => typeof t === "string");
-            return { relevant, suggestedTerms };
-        }
-        return { relevant: false, suggestedTerms: [] };
-    }
-    catch (err) {
-        log(`⚠️ [searchFiles] Failed to check FTS relevance: ${String(err)}`);
-        return { relevant: false, suggestedTerms: [] };
-    }
-}
-;
 async function expandQueryWithModel(query) {
     const prompt = `
 You are assisting a code search system.
@@ -305,25 +329,3 @@ Question:
         return [];
     }
 }
-async function semanticFallbackSearch(query, topK) {
-    const db = getDbForRepo();
-    const seen = new Map();
-    const llmTerms = await expandQueryWithModel(query);
-    logInputOutput("semanticSearchFiles blind expansion", "output", {
-        query,
-        suggestedTerms: llmTerms,
-    });
-    for (const term of llmTerms) {
-        const safeTerm = sanitizeQueryForFts(term);
-        const rows = db
-            .prepare(sqlTemplates.searchFilesTemplate)
-            .all(safeTerm, RELATED_FILES_LIMIT);
-        for (const row of rows) {
-            if (!seen.has(row.id))
-                seen.set(row.id, row);
-        }
-    }
-    if (seen.size === 0)
-        return [];
-    return rankAndMap(seen, topK);
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.163",
+  "version": "0.1.164",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"