paqad-ai 1.55.0 → 1.56.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # paqad-ai
 
+## 1.56.0
+
+### Minor Changes
+
+- 997bbc0: RAG retrieval was dark: local-model cosine scores on hybrid-fused results rarely reached the 0.75 precision floor, so every query fell back below-floor and the model never received a slice. Retrieval now uses floor-with-relief — the 0.75 floor still marks high-confidence hits, but when nothing clears it the top slices at or above a new `rag_relief_floor` (default 0.35, chosen from live probe data) are delivered tagged low-confidence instead of nothing. When retrieval still finds nothing, the session-context artifact carries one honest line naming the best score instead of silently omitting the section.
+
+  Also wires the previously-defined-but-never-recorded `used` RAG-evidence event: the background context worker now records what it actually delivered into the artifact (injected sections, slice/pointer counts, top score, bytes) into the per-feature `.paqad/ledger/feature-evidence/<feature>/rag.jsonl`, so you can see whether RAG was used — not just that retrieval ran. Adds a `paqad-ai rag probe "<query>"` verb that prints pre-floor top-10 fused scores for diagnosis.
+
 ## 1.55.0
 
 ### Minor Changes

package/dist/cli/index.js

@@ -344,6 +344,11 @@ function defaultIntelligenceConfig() {
   return {
     rag_enabled: false,
     rag_similarity_threshold: 0.75,
+    // Issue #354 — chosen from live probe data on this repo: genuine on-target queries
+    // top out at 0.37-0.52 (local MiniLM cosine on hybrid-fused results), while a truly
+    // irrelevant query ("kubernetes ingress") tops at 0.32. 0.35 sits between: it
+    // delivers the weakest real hit and darkens the negative control.
+    rag_relief_floor: 0.35,
     rag_top_n: 20,
     rag_max_file_size: 153600,
     benchmark_gates: { ...DEFAULT_BENCHMARK_GATES },
@@ -368,6 +373,7 @@ function normalizeIntelligenceConfig(input3) {
     embedding_provider: provider,
     embedding_model: input3.embedding_model ?? (provider ? getDefaultEmbeddingModel(provider) : void 0),
     rag_similarity_threshold: input3.rag_similarity_threshold ?? defaults.rag_similarity_threshold,
+    rag_relief_floor: input3.rag_relief_floor ?? defaults.rag_relief_floor,
     rag_top_n: input3.rag_top_n ?? defaults.rag_top_n,
     rag_max_file_size: input3.rag_max_file_size ?? defaults.rag_max_file_size,
     rag_base_branch: input3.rag_base_branch ?? defaults.rag_base_branch,
@@ -648,6 +654,7 @@ function resolveFrameworkConfigFromMap(raw) {
     embedding_provider: provider,
     embedding_model: embeddingModel,
     rag_similarity_threshold: rn("rag_similarity_threshold"),
+    rag_relief_floor: rn("rag_relief_floor"),
     rag_top_n: rn("rag_top_n"),
     rag_max_file_size: rn("rag_max_file_size"),
     rag_base_branch: raw.get("rag_base_branch")?.trim() || void 0
@@ -1008,6 +1015,7 @@ function frameworkOverridesToFlat(overrides) {
       i.rag_similarity_threshold,
       d.intelligence.rag_similarity_threshold
     );
+    put("rag_relief_floor", i.rag_relief_floor, d.intelligence.rag_relief_floor);
     put("rag_top_n", i.rag_top_n, d.intelligence.rag_top_n);
     put("rag_max_file_size", i.rag_max_file_size, d.intelligence.rag_max_file_size);
     put("rag_base_branch", i.rag_base_branch, d.intelligence.rag_base_branch);
@@ -1309,6 +1317,15 @@ var init_framework_config = __esm({
         section: "Intelligence / RAG",
         comment: "Minimum cosine similarity for a chunk to be retrieved."
       },
+      {
+        key: "rag_relief_floor",
+        env: "PAQAD_RAG_RELIEF_FLOOR",
+        type: "number",
+        default: 0.35,
+        group: "rag",
+        section: "Intelligence / RAG",
+        comment: "Relief band for floor-with-relief. When nothing clears rag_similarity_threshold, the top slices at or above this lower score are delivered tagged low-confidence."
+      },
       {
         key: "rag_top_n",
         env: "PAQAD_RAG_TOP_N",
@@ -1586,6 +1603,7 @@ var init_framework_config = __esm({
           "rag_embedding_provider",
           "rag_embedding_model",
           "rag_similarity_threshold",
+          "rag_relief_floor",
           "rag_top_n",
           "rag_max_file_size",
           "rag_base_branch"
@@ -3578,6 +3596,7 @@ var init_project_profile_schema = __esm({
             },
             embedding_model: { type: "string" },
             rag_similarity_threshold: { type: "number" },
+            rag_relief_floor: { type: "number" },
             rag_top_n: { type: "integer", minimum: 1 },
             rag_max_file_size: { type: "integer", minimum: 1 },
             rag_base_branch: { type: "string" },
@@ -10403,6 +10422,21 @@ import { existsSync as existsSync18, statSync as statSync3 } from "fs";
 import { mkdir as mkdir9, readdir, rename as rename3, rm as rm3 } from "fs/promises";
 import { homedir as homedir7 } from "os";
 import { extname, join as join36, resolve as resolve3, sep as sep2 } from "path";
+function toRetrievalResult(candidates, extra) {
+  return {
+    vector_scores: new Map(candidates.map((candidate) => [candidate.id, candidate.score])),
+    chunks_retrieved: candidates.length,
+    retrieved_chunk_ids: candidates.map((candidate) => candidate.id),
+    retrieved_source_files: candidates.map((candidate) => candidate.source_file),
+    retrieved_chunks: candidates.map((candidate) => ({
+      id: candidate.id,
+      source_file: candidate.source_file,
+      content: candidate.content
+    })),
+    best_score: extra.bestScore,
+    low_confidence: extra.lowConfidence
+  };
+}
 function queryTextFromTask(taskDescription, keywords = [], targetFilePath, symbols = []) {
   const parts = [taskDescription, ...keywords, targetFilePath, ...symbols].filter(Boolean);
   return parts.join("\n");
@@ -10450,7 +10484,7 @@ function fuseHybridRanking(dense, queryText) {
   const fused = reciprocalRankFusion([denseOrder, lexical]);
   return fused.map((entry) => byId.get(entry.id)).filter((hit) => hit !== void 0);
 }
-var VISION_MAX_CHUNK_CHARS, CRS_EMBED_BATCH_SIZE, HYBRID_FUSION_POOL_MULTIPLIER, CRS_REVERT_RETENTION_MS, PARTIAL_VECTOR_INDEX, PARTIAL_VECTOR_META, RagService;
+var VISION_MAX_CHUNK_CHARS, CRS_EMBED_BATCH_SIZE, HYBRID_FUSION_POOL_MULTIPLIER, RELIEF_SLICE_CAP, CRS_REVERT_RETENTION_MS, PARTIAL_VECTOR_INDEX, PARTIAL_VECTOR_META, RagService;
 var init_service = __esm({
   "src/rag/service.ts"() {
     "use strict";
@@ -10484,6 +10518,7 @@ var init_service = __esm({
     VISION_MAX_CHUNK_CHARS = 2e3;
     CRS_EMBED_BATCH_SIZE = 32;
     HYBRID_FUSION_POOL_MULTIPLIER = 4;
+    RELIEF_SLICE_CAP = 2;
     CRS_REVERT_RETENTION_MS = 24 * 60 * 60 * 1e3;
     PARTIAL_VECTOR_INDEX = PATHS.VECTOR_INDEX.replace(/\.json$/, ".partial.json");
     PARTIAL_VECTOR_META = PATHS.VECTOR_META.replace(/\.json$/, ".partial.json");
@@ -10789,59 +10824,30 @@ var init_service = __esm({
           if (!skipSync) {
             await this.syncVectorIndex(syncResult, intelligence);
           }
-          const provider = await this.providerFactory(this.projectRoot, intelligence);
-          const [queryVector] = await provider.embed(
-            queryTextFromTask(
-              input3.taskDescription,
-              input3.keywords,
-              input3.targetFilePath,
-              input3.symbolReferences ?? []
-            )
-          );
           const limit = topN ?? intelligence.rag_top_n;
-          const pool = limit * HYBRID_FUSION_POOL_MULTIPLIER;
-          const [fileResults, visionResults] = await Promise.all([
-            this.vectorIndex.query(this.projectRoot, queryVector, pool),
-            this.visionVectorIndex.query(this.projectRoot, queryVector, pool)
-          ]);
-          const dense = [...fileResults, ...visionResults].sort(
-            (left, right) => right.score - left.score
-          );
-          const queryText = queryTextFromTask(
-            input3.taskDescription,
-            input3.keywords,
-            input3.targetFilePath,
-            input3.symbolReferences ?? []
+          const scored = await this.scoreCandidates(input3, limit, intelligence);
+          const bestScore = scored.length > 0 ? scored[0].score : void 0;
+          const aboveFloor = scored.filter(
+            (candidate) => candidate.score >= intelligence.rag_similarity_threshold
           );
-          const fused = fuseHybridRanking(dense, queryText);
-          const reranked = await this.applyReranking(queryText, fused, intelligence.reranking);
-          const results = reranked.slice(0, limit);
-          const filtered = results.filter(
-            (result) => result.score >= intelligence.rag_similarity_threshold
-          );
-          if (filtered.length === 0) {
-            appendRagAudit(this.projectRoot, "WARN", "rag-fallback", {
-              reason: "below-similarity-threshold"
-            });
-            return {
-              vector_scores: /* @__PURE__ */ new Map(),
-              chunks_retrieved: 0,
-              retrieved_chunk_ids: [],
-              retrieved_source_files: [],
-              retrieved_chunks: [],
-              fallback_reason: "below-similarity-threshold"
-            };
+          if (aboveFloor.length > 0) {
+            return toRetrievalResult(aboveFloor, { bestScore, lowConfidence: false });
+          }
+          const relief = scored.filter((candidate) => candidate.score >= intelligence.rag_relief_floor).slice(0, RELIEF_SLICE_CAP);
+          if (relief.length > 0) {
+            return toRetrievalResult(relief, { bestScore, lowConfidence: true });
           }
+          appendRagAudit(this.projectRoot, "WARN", "rag-fallback", {
+            reason: "below-similarity-threshold"
+          });
           return {
-            vector_scores: new Map(filtered.map((result) => [result.item.id, result.score])),
-            chunks_retrieved: filtered.length,
-            retrieved_chunk_ids: filtered.map((result) => result.item.id),
-            retrieved_source_files: filtered.map((result) => result.item.source_file),
-            retrieved_chunks: filtered.map((result) => ({
-              id: result.item.id,
-              source_file: result.item.source_file,
-              content: result.item.content
-            }))
+            vector_scores: /* @__PURE__ */ new Map(),
+            chunks_retrieved: 0,
+            retrieved_chunk_ids: [],
+            retrieved_source_files: [],
+            retrieved_chunks: [],
+            fallback_reason: "below-similarity-threshold",
+            best_score: bestScore
           };
         } catch (error) {
           appendRagAudit(this.projectRoot, "WARN", "rag-fallback", {
@@ -10865,6 +10871,63 @@ var init_service = __esm({
         const syncResult = await this.refreshContext();
         return this.retrieveWithSyncPolicy(syncResult, input3, topN, true);
       }
+      /**
+       * Embed the query, query the file + vision indexes over a widened pool, fuse the
+       * dense and lexical (BM25) legs, optionally rerank, and return the top-`limit`
+       * candidates with their scores — BEFORE any similarity/relief floor is applied.
+       *
+       * This is the single canonical scoring path (RULE-04a1): both floor-with-relief
+       * retrieval ({@link retrieveWithSyncPolicy}) and the `probe` diagnostic call it, so
+       * a probe reports exactly the scores retrieval sees. Assumes the index is present
+       * and valid — callers check {@link getStatus} first.
+       */
+      async scoreCandidates(input3, limit, intelligence) {
+        const provider = await this.providerFactory(this.projectRoot, intelligence);
+        const queryText = queryTextFromTask(
+          input3.taskDescription,
+          input3.keywords,
+          input3.targetFilePath,
+          input3.symbolReferences ?? []
+        );
+        const [queryVector] = await provider.embed(queryText);
+        const pool = limit * HYBRID_FUSION_POOL_MULTIPLIER;
+        const [fileResults, visionResults] = await Promise.all([
+          this.vectorIndex.query(this.projectRoot, queryVector, pool),
+          this.visionVectorIndex.query(this.projectRoot, queryVector, pool)
+        ]);
+        const dense = [...fileResults, ...visionResults].sort(
+          (left, right) => right.score - left.score
+        );
+        const fused = fuseHybridRanking(dense, queryText);
+        const reranked = await this.applyReranking(queryText, fused, intelligence.reranking);
+        return reranked.slice(0, limit).map((result) => ({
+          id: result.item.id,
+          source_file: result.item.source_file,
+          content: result.item.content,
+          score: result.score
+        }));
+      }
+      /**
+       * Issue #354 diagnostic — return the top-`limit` scored candidates for a query
+       * BEFORE the similarity/relief floor, so `paqad-ai rag probe` can show how far real
+       * scores sit from the floor on a live repo (the gap that made retrieval dark).
+       * Refreshes the index internally like {@link retrieveForEval}. Returns `[]` when rag
+       * is off or the index is missing/invalid.
+       */
+      async probe(input3, topN) {
+        const intelligence = normalizeIntelligenceConfig(
+          readProjectProfile(this.projectRoot)?.intelligence
+        );
+        if (!intelligence.rag_enabled || !intelligence.embedding_provider) {
+          return [];
+        }
+        await this.refreshContext();
+        const status = await this.getStatus();
+        if (!status.index_present || !status.valid) {
+          return [];
+        }
+        return this.scoreCandidates(input3, topN ?? intelligence.rag_top_n, intelligence);
+      }
       /**
        * Accept plain text a consumer extracted from an image (via OCR, captioning,
        * etc.) into the retrieval index. The engine never reads the image itself; it
@@ -29306,7 +29369,7 @@ init_cancelled_error();
 init_events();
 
 // src/index.ts
-var VERSION = "1.55.0";
+var VERSION = "1.56.0";
 
 // src/cli/commands/audit.ts
 init_esm_shims();
@@ -40710,9 +40773,6 @@ function truncateSlice(content) {
   return `${body.slice(0, MAX_SLICE_CHARS)}
 \u2026[slice truncated at ${MAX_SLICE_CHARS} chars]`;
 }
-function applyPrecisionFloor(slices, floor) {
-  return slices.filter((slice) => typeof slice.score === "number" && slice.score >= floor);
-}
 function formatScore(score) {
   if (typeof score !== "number") {
     return "";
@@ -40730,11 +40790,19 @@ function dedupeSlices(slices) {
   }
   return unique;
 }
-function composeRetrievalSection(slices) {
+function composeRetrievalSection(slices, opts = {}) {
   if (slices.length === 0) {
+    if (typeof opts.bestScore === "number") {
+      return `## Retrieved context \u2014 none above the confidence floor (best match ${Math.round(
+        opts.bestScore * 100
+      )}%)
+> The index had no slice confident enough to inject for the files in play; read the live files directly.
+`;
+    }
     return "";
   }
   const capped = dedupeSlices(slices).slice(0, MAX_RETRIEVAL_SLICES);
+  const lowConfidence = capped.some((slice) => slice.lowConfidence);
   const blocks = capped.map(
     (slice) => `### ${slice.source_file}${formatScore(slice.score)}
 \`\`\`
@@ -40744,8 +40812,10 @@ ${truncateSlice(
 \`\`\``
   ).join("\n\n");
   const noun = capped.length === 1 ? "slice" : "slices";
-  return `## Retrieved context \u2014 ${capped.length} ${noun} relevant to the files in play
-> Advisory hints retrieved from the index, not ground truth. Re-read the live files before relying on them; the match % is the index's confidence, not correctness.
+  const heading = lowConfidence ? `## Retrieved context \u2014 ${capped.length} low-confidence ${noun} (nothing cleared the confidence floor)` : `## Retrieved context \u2014 ${capped.length} ${noun} relevant to the files in play`;
+  const guidance = lowConfidence ? `> Weak matches, below the confidence floor \u2014 treat as loose leads only and verify against the live files before relying on them.` : `> Advisory hints retrieved from the index, not ground truth. Re-read the live files before relying on them; the match % is the index's confidence, not correctness.`;
+  return `${heading}
+${guidance}
 
 ${blocks}
 `;
@@ -40775,7 +40845,7 @@ async function gatherWorkingSetSlices(projectRoot, options = {}) {
   const changedPaths = options.changedPaths ?? (await loadChangeEvidence(projectRoot)).files;
   const promptQuery = options.query?.trim();
   if (changedPaths.length === 0 && !promptQuery) {
-    return [];
+    return { slices: [], bestScore: null };
   }
   const intelligence = normalizeIntelligenceConfig(readProjectProfile(projectRoot)?.intelligence);
   let effectiveTopN = options.topN;
@@ -40783,7 +40853,7 @@ async function gatherWorkingSetSlices(projectRoot, options = {}) {
     const routing = options.routing ?? { scope: deriveScopeFromWorkingSet(changedPaths) };
     const gate = gateRetrieval({ ...routing, baseTopN: intelligence.rag_top_n });
     if (gate.skip) {
-      return [];
+      return { slices: [], bestScore: null };
     }
     effectiveTopN = gate.topN;
   }
@@ -40793,15 +40863,15 @@ async function gatherWorkingSetSlices(projectRoot, options = {}) {
   try {
     result = await service.retrieveForEval(retrievalInput, effectiveTopN);
   } catch {
-    return [];
+    return { slices: [], bestScore: null };
   }
   const slices = result.retrieved_chunks.map((chunk) => ({
     source_file: chunk.source_file,
     content: chunk.content,
-    score: result.vector_scores.get(chunk.id)
+    score: result.vector_scores.get(chunk.id),
+    lowConfidence: result.low_confidence === true
   }));
-  const floor = options.precisionFloor ?? intelligence.rag_similarity_threshold;
-  const aboveFloor = applyPrecisionFloor(slices, floor);
+  const bestScore = typeof result.best_score === "number" ? result.best_score : null;
   const scope = options.scope ?? scopeForWorkflow(options.routing?.workflow);
   if (options.recordEvidence) {
     recordRagEvidence(
@@ -40819,7 +40889,7 @@ async function gatherWorkingSetSlices(projectRoot, options = {}) {
       }
     );
   }
-  return filterToScope(aboveFloor, scope);
+  return { slices: filterToScope(slices, scope), bestScore };
 }
 
 // src/pipeline/session-route.ts
@@ -40858,6 +40928,9 @@ function readSessionRoute(projectRoot) {
   }
 }
 
+// src/cli/commands/rag.ts
+init_recorder();
+
 // src/rag/background-sync.ts
 init_esm_shims();
 import { join as join196 } from "path";
@@ -41465,6 +41538,42 @@ function createRagCommand() {
     process.stdout.write(`${JSON.stringify(await service.getStatus(), null, 2)}
 `);
   });
+  command.command("probe <query>").description(
+    "Print the top pre-floor retrieval scores for a query (diagnostic; no floor filter)"
+  ).option("--project-root <path>", "Project root", process.cwd()).option("--top-n <n>", "How many candidates to show", "10").action(async (query, options) => {
+    const service = new RagService(options.projectRoot);
+    const { intelligence } = resolveFrameworkConfig(options.projectRoot);
+    const floor = intelligence.rag_similarity_threshold;
+    const reliefFloor = intelligence.rag_relief_floor;
+    const topN = Number.parseInt(options.topN, 10);
+    const candidates = await service.probe(
+      { taskDescription: query, keywords: [] },
+      Number.isFinite(topN) && topN > 0 ? topN : 10
+    );
+    const rows = candidates.map((candidate, index) => ({
+      rank: index + 1,
+      source_file: candidate.source_file,
+      score: Number(candidate.score.toFixed(4)),
+      above_floor: candidate.score >= floor,
+      above_relief: candidate.score >= reliefFloor,
+      gap_to_floor: Number((floor - candidate.score).toFixed(4))
+    }));
+    process.stdout.write(
+      `${JSON.stringify(
+        {
+          query,
+          similarity_threshold: floor,
+          relief_floor: reliefFloor,
+          candidates: rows.length,
+          best_score: rows[0]?.score ?? null,
+          rows
+        },
+        null,
+        2
+      )}
+`
+    );
+  });
   command.command("refresh-context").description(
     "Sync the index, retrieve slices, and recompose session context (background worker)"
   ).option("--project-root <path>", "Project root", process.cwd()).option("--quiet", "Suppress output (used by the background trigger)").action(async (options) => {
@@ -41486,24 +41595,24 @@ function createRagCommand() {
       return;
     }
     const sync = await backgroundIndexSync(options.projectRoot);
-    const slices = retrieves ? await gatherWorkingSetSlices(options.projectRoot, {
+    const { slices, bestScore } = retrieves ? await gatherWorkingSetSlices(options.projectRoot, {
       recordEvidence: { adapter: "engine" },
       query: route?.query
-    }) : [];
-    const retrievalSection = slices.length > MAX_RETRIEVAL_SLICES ? composeContextPack(
-      distillSlices(slices, {
-        readFile: (path11) => {
-          try {
-            return readFileSync99(
-              isAbsolute10(path11) ? path11 : join198(options.projectRoot, path11),
-              "utf8"
-            );
-          } catch {
-            return void 0;
-          }
+    }) : { slices: [], bestScore: null };
+    const usesContextPack = slices.length > MAX_RETRIEVAL_SLICES;
+    const packEntries = usesContextPack ? distillSlices(slices, {
+      readFile: (path11) => {
+        try {
+          return readFileSync99(
+            isAbsolute10(path11) ? path11 : join198(options.projectRoot, path11),
+            "utf8"
+          );
+        } catch {
+          return void 0;
         }
-      })
-    ) : composeRetrievalSection(slices);
+      }
+    }) : [];
+    const retrievalSection = usesContextPack ? composeContextPack(packEntries) : composeRetrievalSection(slices, { bestScore });
     const memorySection = retrieves ? gatherCodebaseMemory(options.projectRoot) : "";
     let driftSection = "";
     if (retrieves) {
@@ -41516,6 +41625,31 @@ function createRagCommand() {
       driftSection,
       loadRules
     });
+    if (retrieves) {
+      const sliceCount = usesContextPack ? 0 : slices.length;
+      const pointerCount = usesContextPack ? packEntries.length : 0;
+      const injectedSections = [];
+      if (loadRules) injectedSections.push("rules");
+      if (sliceCount > 0 || pointerCount > 0) injectedSections.push("retrieval");
+      if (memorySection) injectedSections.push("memory");
+      if (driftSection) injectedSections.push("drift");
+      recordRagEvidence(
+        options.projectRoot,
+        "used",
+        {
+          injected: sliceCount > 0 || pointerCount > 0 || Boolean(memorySection) || Boolean(driftSection),
+          injected_sections: injectedSections,
+          slice_count: sliceCount,
+          pointer_count: pointerCount,
+          score_top: bestScore,
+          bytes_injected: Buffer.byteLength(
+            `${retrievalSection}${memorySection}${driftSection}`,
+            "utf8"
+          )
+        },
+        { ragEnabled: true, adapter: "engine" }
+      );
+    }
     if (!options.quiet) {
       process.stdout.write(
         `${target ? `wrote ${target}` : "nothing to compose"}; index sync: ${sync.synced ? "done" : sync.reason}; slices: ${slices.length}