sift-light 1.0.0

package/src/evidence-candidates.ts

@@ -0,0 +1,344 @@
+import { resolve } from "node:path";
+import {
+  MAX_ANALYSIS_RESULTS,
+  MAX_STRUCTURE_BYTES,
+  MAX_STRUCTURE_FILES,
+} from "./analysis-limits.js";
+import { consumeCappedLines } from "./capped-lines.js";
+import { abortError, SiftLightError } from "./errors.js";
+import { readGitChanges, type GitChangeRequest } from "./git-source.js";
+import { filterHistoricalPaths } from "./historical-paths.js";
+import { runOwnedProcess } from "./owned-process.js";
+import { isPathInsideCwd } from "./path-policy.js";
+import { patternArguments, type RipgrepRunner } from "./rg.js";
+import { resolveRipgrepExecutable } from "./ripgrep-executable.js";
+import { sameSourceRevision } from "./source.js";
+import { SourceBudgetError } from "./source-access.js";
+import {
+  SourceDocument,
+  SourceDocumentError,
+  type ByteRange,
+  type SourceReference,
+} from "./source-document.js";
+import { MAX_PROTOCOL_LINE_BYTES, type SearchRequest } from "./types.js";
+
+export interface EvidenceCandidateFile {
+  document: SourceDocument;
+  occurrences: ByteRange[];
+  changedRanges?: ByteRange[];
+  change?: string;
+}
+
+export interface EvidenceCandidateOptions {
+  request: SearchRequest;
+  changes?: GitChangeRequest;
+  cwd: string;
+  signal?: AbortSignal;
+  access: { load(path: string, expected?: SourceReference): Promise<SourceDocument> };
+  runRipgrep: RipgrepRunner;
+  maxFiles?: number;
+}
+
+export interface EvidenceCandidates {
+  files: EvidenceCandidateFile[];
+  partial: boolean;
+  reasons: string[];
+  filesRead: number;
+  bytesRead: number;
+  changes?: { base: string; target: string; scope: string; side: string };
+}
+
+interface MatchBudget {
+  retained: number;
+  protocolBytes: number;
+}
+class CandidateLimit extends SiftLightError {}
+
+function record(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function integer(value: unknown): value is number {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+
+function eventBytes(value: unknown): Buffer {
+  if (record(value) && typeof value.text === "string") return Buffer.from(value.text);
+  if (record(value) && typeof value.bytes === "string") return Buffer.from(value.bytes, "base64");
+  throw new SiftLightError("Raw ripgrep event omitted source bytes");
+}
+
+/** Half-open source ranges never attribute the start of an unchanged next line to a changed line. */
+export function occurrenceInsideRanges(
+  range: ByteRange,
+  allowed: ByteRange[],
+  document: SourceDocument,
+): boolean {
+  return allowed.some(
+    (outer) =>
+      range.start >= outer.start &&
+      range.end <= outer.end &&
+      (range.end > range.start ||
+        range.start < outer.end ||
+        (outer.end === document.bytes.length && document.bytes.at(-1) !== 10)),
+  );
+}
+
+async function searchRawSource(
+  cwd: string,
+  document: SourceDocument,
+  request: SearchRequest,
+  budget: MatchBudget,
+  allowed: ByteRange[] | undefined,
+  signal?: AbortSignal,
+): Promise<{ occurrences: ByteRange[]; reason?: string }> {
+  const occurrences: ByteRange[] = [];
+  try {
+    const result = await runOwnedProcess(
+      {
+        executable: await resolveRipgrepExecutable(),
+        args: [
+          "--no-config",
+          "--encoding",
+          "none",
+          "--json",
+          "--line-number",
+          "--color=never",
+          ...patternArguments(request),
+          "--",
+          request.pattern,
+          "-",
+        ],
+        cwd,
+        input: document.bytes,
+        ...(signal ? { signal } : {}),
+      },
+      (stdout) =>
+        consumeCappedLines(
+          stdout,
+          (line) => {
+            budget.protocolBytes += Buffer.byteLength(line);
+            if (budget.protocolBytes > MAX_STRUCTURE_BYTES)
+              throw new CandidateLimit("Raw candidate matching reached the 32 MiB protocol budget");
+            let event: unknown;
+            try {
+              event = JSON.parse(line);
+            } catch (error) {
+              throw new SiftLightError("Invalid raw ripgrep JSON", { cause: error });
+            }
+            if (!record(event) || event.type !== "match") return;
+            const data = event.data;
+            if (
+              !record(data) ||
+              !integer(data.absolute_offset) ||
+              !integer(data.line_number) ||
+              data.line_number < 1 ||
+              !Array.isArray(data.submatches)
+            )
+              throw new SiftLightError("Invalid raw ripgrep match event");
+            const start = data.absolute_offset;
+            const bytes = eventBytes(data.lines);
+            if (
+              document.lineStarts[data.line_number - 1] !== start ||
+              start + bytes.length > document.bytes.length ||
+              !document.bytes.subarray(start, start + bytes.length).equals(bytes)
+            )
+              throw new SiftLightError(
+                "Raw ripgrep evidence does not match its source version and line offset",
+              );
+            for (const submatch of data.submatches) {
+              if (
+                !record(submatch) ||
+                !integer(submatch.start) ||
+                !integer(submatch.end) ||
+                submatch.end < submatch.start ||
+                submatch.end > bytes.length ||
+                !bytes.subarray(submatch.start, submatch.end).equals(eventBytes(submatch.match))
+              )
+                throw new SiftLightError("Invalid raw ripgrep occurrence bounds or bytes");
+              const range = { start: start + submatch.start, end: start + submatch.end };
+              if (allowed && !occurrenceInsideRanges(range, allowed, document)) continue;
+              if (budget.retained >= MAX_ANALYSIS_RESULTS)
+                throw new CandidateLimit(
+                  `Candidate matching reached the ${String(MAX_ANALYSIS_RESULTS)} occurrence limit`,
+                );
+              budget.retained += 1;
+              occurrences.push(range);
+            }
+          },
+          { maxLineBytes: MAX_PROTOCOL_LINE_BYTES },
+        ),
+    );
+    if (result.code !== 0 && result.code !== 1)
+      throw new SiftLightError(result.stderr.trim() || `Raw ripgrep exited ${String(result.code)}`);
+  } catch (error) {
+    if (!(error instanceof CandidateLimit)) throw error;
+    return { occurrences, reason: error.message };
+  }
+  return { occurrences };
+}
+
+/* oxlint-disable no-await-in-loop -- retained files share one source and occurrence budget. */
+async function ordinaryCandidates(options: EvidenceCandidateOptions): Promise<EvidenceCandidates> {
+  const scan = await options.runRipgrep(options.request, options.cwd, options.signal);
+  if (options.signal?.aborted) throw abortError();
+  const reasons = new Set<string>();
+  if (!scan.snapshotComplete) {
+    reasons.add("Search retention is partial; only retained matching files can be analyzed");
+    for (const reason of scan.retention?.reasons ?? []) reasons.add(reason);
+  }
+  const grouped = new Map<string, typeof scan.matches>();
+  for (const match of scan.matches) {
+    const existing = grouped.get(match.absolutePath);
+    if (existing) existing.push(match);
+    else grouped.set(match.absolutePath, [match]);
+  }
+  const files: EvidenceCandidateFile[] = [];
+  let filesRead = 0;
+  let bytesRead = 0;
+  let retained = 0;
+  const maxFiles = options.maxFiles ?? MAX_STRUCTURE_FILES;
+  for (const [absolute, matches] of grouped) {
+    if (options.signal?.aborted) throw abortError();
+    const revision = scan.sourceRevisions.get(absolute);
+    if (!revision) {
+      reasons.add("Some matching files lack a verified search revision");
+      continue;
+    }
+    if (filesRead >= maxFiles || bytesRead + revision.size > MAX_STRUCTURE_BYTES) {
+      reasons.add(`Candidate analysis reached the ${String(maxFiles)}-file / 32 MiB source limit`);
+      continue;
+    }
+    filesRead += 1;
+    let document: SourceDocument;
+    try {
+      document = await options.access.load(absolute);
+    } catch (error) {
+      if (error instanceof SourceDocumentError || error instanceof SourceBudgetError) {
+        reasons.add(error.message);
+        continue;
+      }
+      throw error;
+    }
+    bytesRead += document.bytes.length;
+    if (
+      document.reference.origin.kind !== "worktree" ||
+      !sameSourceRevision(revision, document.reference.origin.revision)
+    ) {
+      reasons.add(`Source changed since search: ${document.path}`);
+      continue;
+    }
+    if (
+      (document.bytes[0] === 255 && document.bytes[1] === 254) ||
+      (document.bytes[0] === 254 && document.bytes[1] === 255)
+    ) {
+      reasons.add(
+        `Transcoded search offsets cannot be bound to raw UTF-16 source: ${document.path}`,
+      );
+      continue;
+    }
+    const utf8Bom = document.bytes.subarray(0, 3).equals(Buffer.from([239, 187, 191]));
+    const occurrences: ByteRange[] = [];
+    for (const match of matches) {
+      const lineStart = document.lineStarts[match.lineNumber - 1];
+      if (lineStart === undefined)
+        throw new SiftLightError("Retained match line is outside its verified source");
+      const base = lineStart + (utf8Bom && match.lineNumber === 1 ? 3 : 0);
+      const lineEnd = document.lineStarts[match.lineNumber] ?? document.bytes.length;
+      if (match.occurrences.length === 0)
+        reasons.add("Some retained matches have no exact occurrence ranges");
+      for (const occurrence of match.occurrences) {
+        const range = { start: base + occurrence.byteStart, end: base + occurrence.byteEnd };
+        document.checkRange(range);
+        if (range.end > lineEnd)
+          throw new SiftLightError("Retained occurrence extends beyond its verified source line");
+        if (retained >= MAX_ANALYSIS_RESULTS) {
+          reasons.add(
+            `Candidate matching reached the ${String(MAX_ANALYSIS_RESULTS)} occurrence limit`,
+          );
+          break;
+        }
+        retained += 1;
+        occurrences.push(range);
+      }
+    }
+    if (occurrences.length > 0) files.push({ document, occurrences });
+  }
+  return { files, partial: reasons.size > 0, reasons: [...reasons], filesRead, bytesRead };
+}
+/* oxlint-enable no-await-in-loop */
+
+/** One candidate boundary for ordinary retained searches and fixed Git source versions. */
+/* oxlint-disable no-await-in-loop -- Git sources consume one cumulative rg protocol and occurrence budget. */
+export async function collectEvidenceCandidates(
+  options: EvidenceCandidateOptions,
+): Promise<EvidenceCandidates> {
+  if (!options.changes) return ordinaryCandidates(options);
+  if (
+    options.request.path &&
+    !isPathInsideCwd(resolve(options.cwd, options.request.path), options.cwd)
+  ) {
+    throw new SiftLightError(
+      "Git changes for paths outside cwd are not supported; relaunch Pi from that repository or a common parent",
+    );
+  }
+  const reasons = new Set<string>();
+  const result = await readGitChanges(options.cwd, options.changes, options.signal, {
+    filterPaths: async (paths) => {
+      const selected = await filterHistoricalPaths(
+        options.cwd,
+        paths,
+        options.request,
+        options.signal,
+      );
+      for (const reason of selected.reasons) reasons.add(reason);
+      return { paths: selected.paths, bytesRead: selected.ignoreBytesRead };
+    },
+  });
+  if (options.signal?.aborted) throw abortError();
+  for (const reason of result.reasons) reasons.add(reason);
+  const files: EvidenceCandidateFile[] = [];
+  const budget: MatchBudget = { retained: 0, protocolBytes: 0 };
+  for (const file of result.files) {
+    if (options.signal?.aborted) throw abortError();
+    if (!file.content || !file.origin) {
+      if (file.sourceStatus !== "absent")
+        reasons.add(`${file.path}: ${file.reason ?? file.sourceStatus}`);
+      continue;
+    }
+    const document = new SourceDocument({ path: file.path, origin: file.origin }, file.content);
+    const changedRanges = file.changedRanges.map((range) =>
+      document.lineRange(range.startLine, range.endLine),
+    );
+    if (options.changes.scope === "lines" && changedRanges.length === 0) continue;
+    const matched = await searchRawSource(
+      options.cwd,
+      document,
+      options.request,
+      budget,
+      options.changes.scope === "lines" ? changedRanges : undefined,
+      options.signal,
+    );
+    if (matched.reason) reasons.add(matched.reason);
+    if (matched.occurrences.length > 0)
+      files.push({
+        document,
+        occurrences: matched.occurrences,
+        changedRanges,
+        change: file.change,
+      });
+    if (matched.reason) {
+      reasons.add("Remaining Git candidate files were not searched after the matching limit");
+      break;
+    }
+  }
+  return {
+    files,
+    partial: reasons.size > 0,
+    reasons: [...reasons],
+    filesRead: result.filesRead,
+    bytesRead: result.bytesRead,
+    changes: { base: result.base, target: result.target, scope: result.scope, side: result.side },
+  };
+}
+/* oxlint-enable no-await-in-loop */

package/src/evidence-ranking.ts

@@ -0,0 +1,48 @@
+import type { AnalysisItem } from "./analysis-types.js";
+
+/** Preserve evidence tiers; within a tier surface one location per file before repeating a file. */
+export function rankEvidence(
+  items: readonly AnalysisItem[],
+  priority: (item: AnalysisItem) => number,
+): AnalysisItem[] {
+  const tiers = new Map<number, Map<string, AnalysisItem[]>>();
+  const ordered = items.toSorted(
+    (a, b) =>
+      priority(a) - priority(b) ||
+      a.path.localeCompare(b.path) ||
+      a.line - b.line ||
+      (a.range?.start ?? 0) - (b.range?.start ?? 0),
+  );
+  for (const item of ordered) {
+    const score = priority(item);
+    let files = tiers.get(score);
+    if (!files) {
+      files = new Map();
+      tiers.set(score, files);
+    }
+    let entries = files.get(item.path);
+    if (!entries) {
+      entries = [];
+      files.set(item.path, entries);
+    }
+    entries.push(item);
+  }
+  const result: AnalysisItem[] = [];
+  for (const files of tiers.values()) {
+    const groups = [...files.values()];
+    const depth = Math.max(0, ...groups.map((group) => group.length));
+    for (let index = 0; index < depth; index += 1)
+      for (const group of groups) {
+        const item = group[index];
+        if (item)
+          result.push({
+            ...item,
+            details: {
+              ...item.details,
+              rankingOrder: "evidence tier, then round-robin files, then source position",
+            },
+          });
+      }
+  }
+  return result;
+}