scai 0.1.176 → 0.1.177

package/dist/commands/FindCmd.js

@@ -1,5 +1,5 @@
 import { queryFiles } from '../db/fileIndex.js';
-import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
+import { compileSearchQuery } from '../utils/compileSearchQuery.js';
 import path from 'path';
 import os from 'os';
 export async function runFindCommand(query) {
@@ -8,8 +8,8 @@ export async function runFindCommand(query) {
         return;
     }
     console.log(`\n🔍 Searching for: "${query}"\n`);
-    const sanitizedQuery = sanitizeQueryForFts(query);
-    const results = queryFiles(sanitizedQuery);
+    const compiled = compileSearchQuery({ query, mode: "fts" });
+    const results = compiled.fts.expression ? queryFiles(compiled.fts.expression) : [];
     if (results.length === 0) {
         console.log('⚠️ No matching files found.');
         return;

package/dist/commands/SummaryCmd.js

@@ -4,7 +4,7 @@ import readline from 'readline';
 import { queryFiles } from '../db/fileIndex.js';
 import { summaryModule } from '../pipeline/modules/summaryModule.js';
 import { detectFileType } from '../fileRules/detectFileType.js';
-import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
+import { compileSearchQuery } from '../utils/compileSearchQuery.js';
 import { styleText } from '../utils/outputFormatter.js';
 import { indexFile } from '../daemon/runIndexingBatch.js';
 export async function summarizeFile(filepath) {
@@ -12,8 +12,8 @@ export async function summarizeFile(filepath) {
     let filePathResolved;
     // 📁 Resolve file path from index or local disk
     if (filepath) {
-        const sanitizedQuery = sanitizeQueryForFts(filepath);
-        const matches = queryFiles(sanitizedQuery);
+        const compiled = compileSearchQuery({ query: filepath, mode: "fts" });
+        const matches = compiled.fts.expression ? queryFiles(compiled.fts.expression) : [];
         if (matches.length > 0) {
             const topMatch = matches[0];
             filePathResolved = path.resolve(process.cwd(), topMatch.path);
@@ -34,7 +34,8 @@ export async function summarizeFile(filepath) {
     }
     // 📄 Load file content (from path or stdin)
     if (filePathResolved) {
-        const matches = queryFiles(`"${filePathResolved}"`);
+        const compiled = compileSearchQuery({ query: filePathResolved, mode: "fts" });
+        const matches = compiled.fts.expression ? queryFiles(compiled.fts.expression) : [];
         const match = matches.find(row => path.resolve(row.path) === filePathResolved);
         if (match?.summary) {
             console.log(`🧠 Cached summary for ${filepath}:\n`);

package/dist/db/fileIndex.js

@@ -15,9 +15,8 @@ import * as sqlTemplates from '../db/sqlTemplates.js';
 import { RELATED_FILES_LIMIT } from '../constants.js';
 import { generate } from '../lib/generate.js';
 import { logInputOutput } from '../utils/promptLogHelper.js';
-import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
+import { compileSearchQuery } from '../utils/compileSearchQuery.js';
 import { extractTaggedContent } from '../utils/parseTaggedContent.js';
-import { extractFileReferences } from '../utils/extractFileReferences.js';
 const QUERY_OPERATOR_TOKENS = new Set(["or", "and", "not", "near"]);
 const GENERIC_FTS_TERMS = new Set([
     "file",
@@ -659,12 +658,14 @@ export async function plannerSearchFiles(originalQuery, query, topK = 5) {
     const db = getDbForRepo();
     const seen = new Map();
     const usedQueries = [];
-    const safeQuery = sanitizeQueryForFts(query);
+    const compiledPrimary = compileSearchQuery({ query, mode: "fts" });
+    const safeQuery = compiledPrimary.fts.expression;
     if (safeQuery)
         usedQueries.push(safeQuery);
-    const primaryResults = db
-        .prepare(sqlTemplates.searchFilesTemplate)
-        .all(safeQuery, RELATED_FILES_LIMIT);
+    const primaryResults = safeQuery
+        ? db.prepare(sqlTemplates.searchFilesTemplate)
+            .all(safeQuery, RELATED_FILES_LIMIT)
+        : [];
     primaryResults.forEach(r => seen.set(r.id, r));
     logInputOutput("plannerSearchFiles primary FTS", "input", {
         safeQuery,
@@ -673,12 +674,12 @@ export async function plannerSearchFiles(originalQuery, query, topK = 5) {
     if (primaryResults.length === 0) {
         const stmt = db.prepare(sqlTemplates.searchFilesTemplate);
         const llmPrimaryQuery = await generatePrimaryFtsQuery(originalQuery);
-        const llmFallbackQueries = await generateFallbackFtsQueries(originalQuery, llmPrimaryQuery || safeQuery);
+        const llmFallbackQueries = await generateFallbackFtsQueries(originalQuery, llmPrimaryQuery || safeQuery || originalQuery);
         const candidateQueries = [];
         const pushCandidate = (q) => {
             if (!q)
                 return;
-            const sanitized = sanitizeQueryForFts(q);
+            const sanitized = compileSearchQuery({ query: q, mode: "fts" }).fts.expression;
             if (!sanitized)
                 return;
             if (sanitized === safeQuery)
@@ -764,8 +765,12 @@ function buildQueryExpansionTerms(originalQuery, queries) {
  * - output: "\"fileIndex.ts\" OR semanticsearchfiles* OR output*"
  */
 function enforceFtsQueryPolicy(userQuery, candidateQuery, maxTerms, intent = {}) {
-    const safe = sanitizeQueryForFts(candidateQuery);
-    const candidateTerms = splitOrTerms(safe);
+    const candidateTerms = compileSearchQuery({
+        query: candidateQuery,
+        intent,
+        mode: "fts",
+        maxTerms,
+    }).fts.terms;
     const prioritizedAnchors = buildAnchorTerms(userQuery, intent);
     const hasAnchors = prioritizedAnchors.length > 0;
     const filtered = candidateTerms.filter(term => {
@@ -777,25 +782,17 @@ function enforceFtsQueryPolicy(userQuery, candidateQuery, maxTerms, intent = {})
         return true;
     });
     const merged = dedupeTerms([...prioritizedAnchors, ...filtered]);
-    return merged.slice(0, maxTerms).join(" OR ");
-}
-/**
- * Splits an OR query into raw terms.
- * Example: "\"fileIndex.ts\" OR semanticsearchfiles* OR output*" ->
- * ["\"fileIndex.ts\"", "semanticsearchfiles*", "output*"]
- */
-function splitOrTerms(query) {
-    return query
-        .split(/\s+OR\s+/i)
-        .map(part => part.trim())
-        .filter(Boolean);
+    return merged
+        .slice(0, maxTerms)
+        .map(term => `${normalizeFtsTerm(term)}*`)
+        .join(" OR ");
 }
 /**
  * Normalizes an FTS term for stable comparisons by removing quotes/wildcards.
  * Example: "\"fileIndex.ts\"" -> "fileindex.ts", "Module*" -> "module"
  */
 function normalizeFtsTerm(term) {
-    return term.replace(/["*]/g, "").toLowerCase().trim();
+    return term.toLowerCase().replace(/[^a-z0-9_]/g, "").trim();
 }
 /**
  * Dedupes terms using normalized keys while preserving first-seen order.
@@ -820,54 +817,10 @@ function dedupeTerms(terms) {
  * - output includes: ["\"fileIndex.ts\"", "semanticsearchfiles*"]
  */
 function buildAnchorTerms(userQuery, intent = {}) {
-    const anchorTerms = [];
-    const explicitFiles = extractFileReferences(userQuery);
-    const targetFiles = Array.isArray(intent.targetFiles)
-        ? dedupeNormalizedStrings(intent.targetFiles)
-        : [];
-    const targetSymbols = Array.isArray(intent.targetSymbols)
-        ? dedupeNormalizedStrings(intent.targetSymbols)
-        : [];
-    for (const fileRef of targetFiles) {
-        for (const matchedFile of extractFileReferences(fileRef)) {
-            const filename = path.basename(matchedFile);
-            if (filename)
-                anchorTerms.push(`"${filename}"`);
-        }
-    }
-    for (const fileRef of explicitFiles) {
-        const filename = path.basename(fileRef);
-        if (filename)
-            anchorTerms.push(`"${filename}"`);
-    }
-    for (const symbol of targetSymbols) {
-        for (const form of buildSymbolSearchForms(symbol)) {
-            const normalized = normalizeFtsTerm(form);
-            if (!normalized)
-                continue;
-            if (QUERY_OPERATOR_TOKENS.has(normalized))
-                continue;
-            if (GENERIC_FTS_TERMS.has(normalized))
-                continue;
-            anchorTerms.push(`${normalized}*`);
-        }
-    }
-    const symbolMatches = userQuery.match(/[A-Za-z_][A-Za-z0-9_]*/g) ?? [];
-    for (const token of symbolMatches) {
-        const isLikelySymbol = /[A-Z]/.test(token) ||
-            token.includes("_") ||
-            token.endsWith("Step") ||
-            token.endsWith("Module");
-        if (!isLikelySymbol)
-            continue;
-        const normalized = token.toLowerCase();
-        if (QUERY_OPERATOR_TOKENS.has(normalized))
-            continue;
-        if (GENERIC_FTS_TERMS.has(normalized))
-            continue;
-        anchorTerms.push(`${normalized}*`);
-    }
-    return dedupeTerms(anchorTerms);
+    const compiled = compileSearchQuery({ query: userQuery, intent, mode: "fts" });
+    const fileTerms = compiled.anchors.files.flatMap(fileRef => compileSearchQuery({ query: path.basename(fileRef), mode: "fts", maxTerms: 8 }).fts.terms);
+    const symbolTerms = compiled.anchors.symbols.flatMap(symbol => compileSearchQuery({ query: symbol, mode: "fts", maxTerms: 8 }).fts.terms);
+    return dedupeTerms([...fileTerms, ...symbolTerms]);
 }
 function dedupeNormalizedStrings(tokens) {
     // Lightweight normalization only (trim + slash normalization + case-insensitive dedupe).
@@ -909,7 +862,7 @@ Question:
         const response = await generate({ content: prompt, query: "" });
         const rawText = String(response.data ?? "");
         const { content } = extractTaggedContent(rawText, "FILE_CONTENT");
-        return sanitizeQueryForFts(content);
+        return compileSearchQuery({ query: content, mode: "fts" }).fts.expression;
     }
     catch {
         return null;

package/dist/pipeline/modules/fileSearchModule.js

@@ -8,6 +8,7 @@ import { getDbForRepo } from "../../db/client.js";
 import { IGNORED_EXTENSIONS } from "../../fileRules/ignoredExtensions.js";
 import { IGNORED_DIR_NAMES, isPathIgnoredByFolderGlobs } from "../../fileRules/ignoredPaths.js";
 import { logInputOutput } from "../../utils/promptLogHelper.js";
+import { compileSearchQuery } from "../../utils/compileSearchQuery.js";
 async function fetchSummariesForPaths(paths) {
     if (paths.length === 0)
         return {};
@@ -66,8 +67,25 @@ export const fileSearchModule = {
             try {
                 const excludeExtArgs = IGNORED_EXTENSIONS.map(ext => `--exclude=*${ext}`);
                 const excludeDirArgs = IGNORED_DIR_NAMES.map(dir => `--exclude-dir=${dir}`);
-                const grepArgs = ["-ril", ...excludeDirArgs, ...excludeExtArgs, subQuery, repoRoot];
-                const stdout = execFileSync("grep", grepArgs, { encoding: "utf8" });
+                const compiledFallback = compileSearchQuery({ query: subQuery, mode: "auto" });
+                const rgExcludeExtArgs = excludeExtArgs.map(arg => {
+                    const value = arg.replace("--exclude=", "");
+                    return `!${value}`;
+                });
+                const rgExcludeDirArgs = excludeDirArgs.map(arg => {
+                    const value = arg.replace("--exclude-dir=", "");
+                    return `!${value}/**`;
+                });
+                const rgArgs = [
+                    "--files-with-matches",
+                    "--no-messages",
+                    ...rgExcludeDirArgs.flatMap(pattern => ["-g", pattern]),
+                    ...rgExcludeExtArgs.flatMap(pattern => ["-g", pattern]),
+                    ...(compiledFallback.regex.enabled ? [] : ["-F"]),
+                    subQuery,
+                    repoRoot,
+                ];
+                const stdout = execFileSync("rg", rgArgs, { encoding: "utf8" });
                 results = stdout
                     .split("\n")
                     .filter(Boolean)

package/dist/pipeline/modules/gatherInfoModule.js

@@ -1,7 +1,7 @@
 // src/pipeline/modules/gatherInfoModule.ts
 import { getDbForRepo } from "../../db/client.js";
 import chalk from "chalk";
-import { sanitizeQueryForFts } from "../../utils/sanitizeQuery.js";
+import { compileSearchQuery } from "../../utils/compileSearchQuery.js";
 import { logInputOutput } from "../../utils/promptLogHelper.js"; // ✅ import logger
 /** Escape % and _ for LIKE queries */
 function sanitizeForLike(input) {
@@ -32,13 +32,15 @@ export const gatherInfoModule = {
             logInputOutput("gatherInfo", "output", stripEmbeddings(emptyOutput)); // ✅
             return emptyOutput;
         }
-        const sanitizedFts = sanitizeQueryForFts(query);
+        const compiled = compileSearchQuery({ query, mode: "fts" });
+        const sanitizedFts = compiled.fts.expression;
         const likeQuery = `%${sanitizeForLike(query)}%`;
         // 🩹 Handle legacy DBs that might not have `functions_extracted`
         let files = [];
         try {
-            files = db
-                .prepare(`
+            if (sanitizedFts) {
+                files = db
+                    .prepare(`
           SELECT f.id, f.path, f.type, f.summary, f.embedding,
                  f.last_modified, f.indexed_at,
                  COALESCE(f.functions_extracted, 0) AS functions_extracted,
@@ -49,12 +51,14 @@ export const gatherInfoModule = {
           ORDER BY f.path ASC
           LIMIT ?;
         `)
-                .all(maxFiles);
+                    .all(maxFiles);
+            }
         }
         catch (err) {
             console.warn(chalk.yellow("⚠️ 'functions_extracted' column missing in files table, running fallback query..."));
-            files = db
-                .prepare(`
+            if (sanitizedFts) {
+                files = db
+                    .prepare(`
           SELECT f.id, f.path, f.type, f.summary, f.embedding,
                  f.last_modified, f.indexed_at, f.processing_status
           FROM files f
@@ -63,7 +67,8 @@ export const gatherInfoModule = {
           ORDER BY f.path ASC
           LIMIT ?;
         `)
-                .all(maxFiles);
+                    .all(maxFiles);
+            }
         }
         const functions = db
             .prepare(`

package/dist/utils/compileSearchQuery.js

@@ -0,0 +1,138 @@
+import path from "path";
+import { STOP_WORDS } from "../fileRules/stopWords.js";
+import { extractFileReferences } from "./extractFileReferences.js";
+const QUERY_OPERATOR_TOKENS = new Set(["or", "and", "not", "near"]);
+const GENERIC_FTS_TERMS = new Set([
+    "file",
+    "files",
+    "code",
+    "source",
+    "repository",
+    "result",
+    "results",
+    "output",
+    "entry",
+    "database",
+    "configuration",
+]);
+const SYMBOL_HINT_SUFFIXES = ["step", "module", "service", "controller", "handler", "manager"];
+export function compileSearchQuery(args) {
+    const rawQuery = String(args.query ?? "").trim();
+    const intent = args.intent ?? {};
+    const mode = args.mode ?? "auto";
+    const maxTerms = Math.max(1, args.maxTerms ?? 12);
+    const fileRefs = dedupeNormalizedStrings([
+        ...extractFileReferences(rawQuery),
+        ...collectIntentFileRefs(intent),
+    ]);
+    const symbolRefs = dedupeNormalizedStrings([
+        ...collectIntentSymbols(intent),
+        ...extractLikelySymbols(rawQuery),
+    ]);
+    const anchorFileTerms = fileRefs.flatMap(ref => toSearchTerms(path.basename(ref)));
+    const anchorSymbolTerms = symbolRefs.flatMap(ref => toSearchTerms(ref));
+    const baseTerms = dedupeNormalizedStrings([
+        ...toSearchTerms(rawQuery),
+        ...anchorFileTerms,
+        ...anchorSymbolTerms,
+    ]);
+    const droppedTerms = [];
+    const filteredTerms = baseTerms.filter(term => {
+        if (term.length < 2) {
+            droppedTerms.push(term);
+            return false;
+        }
+        if (STOP_WORDS.has(term) || QUERY_OPERATOR_TOKENS.has(term)) {
+            droppedTerms.push(term);
+            return false;
+        }
+        return true;
+    });
+    const hasAnchors = anchorFileTerms.length > 0 || anchorSymbolTerms.length > 0;
+    const ftsTerms = dedupeNormalizedStrings(filteredTerms.filter(term => !(hasAnchors && GENERIC_FTS_TERMS.has(term)))).slice(0, maxTerms);
+    const literalTerms = dedupeNormalizedStrings([
+        rawQuery,
+        ...fileRefs,
+        ...symbolRefs,
+        ...filteredTerms,
+    ]).filter(Boolean);
+    const regexEnabled = mode === "regex" || (mode === "auto" && looksLikeRegex(rawQuery));
+    return {
+        rawQuery,
+        mode,
+        anchors: {
+            files: fileRefs,
+            symbols: symbolRefs,
+        },
+        fts: {
+            terms: ftsTerms,
+            expression: ftsTerms.length > 0 ? ftsTerms.map(term => `${term}*`).join(" OR ") : null,
+        },
+        literal: {
+            terms: literalTerms,
+        },
+        regex: {
+            terms: regexEnabled ? [rawQuery].filter(Boolean) : [],
+            enabled: regexEnabled,
+        },
+        diagnostics: {
+            droppedTerms,
+        },
+    };
+}
+function collectIntentFileRefs(intent) {
+    if (!Array.isArray(intent.targetFiles))
+        return [];
+    return intent.targetFiles
+        .map(entry => String(entry ?? "").trim())
+        .filter(Boolean);
+}
+function collectIntentSymbols(intent) {
+    if (!Array.isArray(intent.targetSymbols))
+        return [];
+    return intent.targetSymbols
+        .map(entry => String(entry ?? "").trim())
+        .filter(Boolean);
+}
+function dedupeNormalizedStrings(tokens) {
+    const out = [];
+    const seen = new Set();
+    for (const token of tokens) {
+        if (typeof token !== "string")
+            continue;
+        const normalized = token.trim().replace(/\\/g, "/");
+        if (!normalized)
+            continue;
+        const key = normalized.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(normalized);
+    }
+    return out;
+}
+function extractLikelySymbols(input) {
+    const matches = String(input ?? "").match(/[A-Za-z_][A-Za-z0-9_]{2,}/g) ?? [];
+    const out = new Set();
+    for (const token of matches) {
+        const lowered = token.toLowerCase();
+        const looksLikeSymbol = /[A-Z]/.test(token) ||
+            token.includes("_") ||
+            SYMBOL_HINT_SUFFIXES.some(suffix => lowered.endsWith(suffix));
+        if (!looksLikeSymbol)
+            continue;
+        out.add(token);
+    }
+    return Array.from(out);
+}
+function toSearchTerms(input) {
+    const matches = String(input ?? "")
+        .toLowerCase()
+        .match(/[a-z0-9_]{2,}/g) ?? [];
+    return Array.from(new Set(matches));
+}
+function looksLikeRegex(input) {
+    if (!input)
+        return false;
+    return /[\[\]{}()+?|\\.^$]/.test(input);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.176",
+  "version": "0.1.177",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"