sift-light 1.0.0

package/src/search-policy.ts

@@ -0,0 +1,102 @@
+import type { SearchKind, ShellLanguage } from "./search-policy-commands.js";
+import { recoverShellSearch } from "./search-policy-recovery.js";
+import { ShellSearchPolicy, type ShellSearchMatch } from "./search-policy-shell.js";
+
+export const SEARCH_POLICY_GUIDANCE =
+  "Local content and filename searches must use sift-light. Built-in search tools and direct search commands are blocked before execution; filtering output from an unrelated producer at a pipeline tail remains available. Use pattern for contents or mode=files with query for filenames. Keep read/edit/write, tests and builds available. After a denial, call sift-light once with the stated repair; do not paste the denial into the request, repeat the blocked call, use another shell/custom script, or weaken the search mode.";
+export const PI_REPLACED_SEARCH_TOOLS = new Set(["grep", "find"]);
+export const PREFERRED_SEARCH_GUIDANCE =
+  "Prefer sift-light for local content and filename searches because it provides bounded evidence, coverage and continuation details. Conventional search entries remain available in advisory mode.";
+const contentTools = new Set(["grep", "Grep", "SearchFileContent"]);
+const fileTools = new Set(["find", "glob", "Glob", "GlobFile", "SearchFiles"]);
+const shellTools = new Set([
+  "bash",
+  "Bash",
+  "powershell",
+  "PowerShell",
+  "Shell",
+  "exec_command",
+  "shell_command",
+  "shell",
+]);
+
+export interface SearchPolicyDecision {
+  block: true;
+  reason: string;
+}
+
+function isInputRecord(input: unknown): input is Record<string, unknown> {
+  return typeof input === "object" && input !== null && !Array.isArray(input);
+}
+
+function recovery(kind: SearchKind): string {
+  return kind === "files"
+    ? '{"mode":"files","path":"<scope>"} (query is optional: omit it to list every file under path, supply it only to match known filename or path text, and use glob for name patterns)'
+    : '{"pattern":"<search text>","path":"<scope>","scope":"strict"}';
+}
+
+function blockedMatch(
+  command: string,
+  match: ShellSearchMatch,
+  workingDirectory: string | undefined,
+): SearchPolicyDecision {
+  const location =
+    match.nestedDepth > 0
+      ? `nested ${match.language} command #${match.commandIndex}`
+      : `${match.language} subcommand #${match.commandIndex}`;
+  const recovered = recoverShellSearch(command, match, workingDirectory);
+  const repair =
+    recovered.kind === "concrete"
+      ? `retry exactly once through sift-light (possibly MCP-prefixed) with ${recovered.request}`
+      : `an equivalent request was not generated because ${recovered.reason}; manually translate the search, then retry exactly once through sift-light (possibly MCP-prefixed) with ${recovery(match.kind)}`;
+  return {
+    block: true,
+    reason: `sift-light search policy blocked direct ${match.kind} search at ${location} (${match.command} …, bytes ${match.startByte}-${match.endByte}); the atomic shell call did not run. Split out non-search operations, then ${repair}. Do not include this denial in the retry, repeat it through another shell/script, or weaken the search. If sift-light is unavailable, report that connection error once without attempting another search.`,
+  };
+}
+
+function blockedTool(kind: SearchKind, toolName: string): SearchPolicyDecision {
+  return {
+    block: true,
+    reason: `sift-light search policy blocked direct ${kind} tool ${toolName}; it did not run. Retry exactly once through sift-light (possibly MCP-prefixed) with ${recovery(kind)}. Do not include this denial in the retry, use another search entry, or weaken the search. If sift-light is unavailable, report that connection error once without attempting another search.`,
+  };
+}
+
+export class SearchPolicy {
+  readonly #shell: ShellSearchPolicy;
+  constructor(assets: URL) {
+    this.#shell = new ShellSearchPolicy(assets);
+  }
+
+  async check(
+    toolName: string,
+    input: unknown,
+    signal?: AbortSignal,
+  ): Promise<SearchPolicyDecision | undefined> {
+    signal?.throwIfAborted();
+    if (contentTools.has(toolName)) return blockedTool("content", toolName);
+    if (fileTools.has(toolName)) return blockedTool("files", toolName);
+    if (!shellTools.has(toolName)) return undefined;
+    if (!isInputRecord(input)) throw new Error("Search policy expected a tool input object");
+    const fields = input;
+    const command = fields.command ?? fields.cmd;
+    if (typeof command !== "string")
+      throw new Error("Search policy expected a shell command string");
+    const shell = typeof fields.shell === "string" ? fields.shell : "";
+    const workingDirectoryValue = fields.workdir ?? fields.cwd;
+    const workingDirectory =
+      workingDirectoryValue === undefined
+        ? process.cwd()
+        : typeof workingDirectoryValue === "string" && workingDirectoryValue.length > 0
+          ? workingDirectoryValue
+          : undefined;
+    const language: ShellLanguage =
+      /powershell|pwsh/iu.test(`${toolName} ${shell}`) ||
+      (process.platform === "win32" && toolName !== "bash")
+        ? "powershell"
+        : "bash";
+    const match = await this.#shell.inspect(command, language);
+    signal?.throwIfAborted();
+    return match ? blockedMatch(command, match, workingDirectory) : undefined;
+  }
+}

package/src/search-retention.ts

@@ -0,0 +1,77 @@
+import { SiftLightError } from "./errors.js";
+import {
+  MAX_SEARCH_STORAGE_BYTES,
+  MAX_STORED_OCCURRENCES,
+  type MatchRecord,
+  type SearchRetentionDetails,
+} from "./types.js";
+
+/** Accounting bounds serialized evidence and occurrence cardinality, not V8 heap bytes. */
+export class SearchRetention {
+  #bytes = 0;
+  #metadataBytes = 0;
+  #metadataLimitReached = false;
+  #occurrences = 0;
+  readonly #reasons = new Set<string>();
+  readonly maxBytes: number;
+  readonly maxOccurrences: number;
+  constructor(maxBytes = MAX_SEARCH_STORAGE_BYTES, maxOccurrences = MAX_STORED_OCCURRENCES) {
+    this.maxBytes = maxBytes;
+    this.maxOccurrences = maxOccurrences;
+    for (const value of [maxBytes, maxOccurrences]) {
+      if (!Number.isSafeInteger(value) || value < 1)
+        throw new SiftLightError("Search retention limits must be positive safe integers");
+    }
+  }
+
+  file(displayPath: string, absolutePath: string): boolean {
+    if (this.#metadataLimitReached) return false;
+    // Reserve the count, revision and path metadata before admitting this file's matches.
+    const bytes = Buffer.byteLength(JSON.stringify([displayPath, absolutePath])) + 512;
+    const metadataLimit = Math.floor(this.maxBytes / 4);
+    if (this.#metadataBytes + bytes > metadataLimit) {
+      this.#metadataLimitReached = true;
+      this.#reasons.add(
+        `File-summary retention reached its ${String(metadataLimit)}-byte share of the search budget; file summaries are partial; narrow the path or filters`,
+      );
+      return false;
+    }
+    this.#bytes += bytes;
+    this.#metadataBytes += bytes;
+    return true;
+  }
+
+  canRetainOccurrences(count: number): boolean {
+    if (this.#occurrences + count <= this.maxOccurrences) return true;
+    this.#reasons.add(`Occurrence retention reached the ${String(this.maxOccurrences)} limit`);
+    return false;
+  }
+
+  retain(match: MatchRecord): boolean {
+    if (!this.canRetainOccurrences(match.occurrences.length)) return false;
+    const bytes = Buffer.byteLength(JSON.stringify(match)) + 1;
+    if (this.#bytes - this.#metadataBytes + bytes > this.maxBytes - Math.floor(this.maxBytes / 4)) {
+      this.#reasons.add(
+        `Search evidence retention reached the ${String(this.maxBytes)}-byte limit`,
+      );
+      return false;
+    }
+    this.#bytes += bytes;
+    this.#occurrences += match.occurrences.length;
+    return true;
+  }
+
+  noteLimit(reason: string): void {
+    this.#reasons.add(reason);
+  }
+
+  get details(): SearchRetentionDetails {
+    return {
+      accountedBytes: this.#bytes,
+      retainedOccurrences: this.#occurrences,
+      maxBytes: this.maxBytes,
+      maxOccurrences: this.maxOccurrences,
+      reasons: [...this.#reasons],
+    };
+  }
+}

package/src/semantic-judge-batches.ts

@@ -0,0 +1,135 @@
+import { SiftLightError } from "./errors.js";
+import type {
+  SemanticJudgeCandidate,
+  SemanticJudgeResult,
+  SemanticJudgeRunner,
+} from "./semantic-judge.js";
+
+const MAX_SEMANTIC_JUDGE_BATCH_CANDIDATES = 8;
+export const MAX_SEMANTIC_JUDGE_REQUEST_BYTES = 64 * 1024;
+
+export class SemanticJudgePayloadTooLargeError extends SiftLightError {
+  constructor(message: string) {
+    super(message);
+    this.name = "SemanticJudgePayloadTooLargeError";
+  }
+}
+
+interface SemanticJudgeBatch {
+  startIndex: number;
+  candidates: SemanticJudgeCandidate[];
+}
+
+export interface SemanticJudgeBatchSuccess extends SemanticJudgeBatch {
+  result: SemanticJudgeResult;
+}
+
+export interface SemanticJudgeBatchFailure extends SemanticJudgeBatch {
+  error: unknown;
+}
+
+export interface SemanticJudgeBatchExecution {
+  successes: SemanticJudgeBatchSuccess[];
+  failures: SemanticJudgeBatchFailure[];
+  attempts: number;
+  splits: number;
+}
+
+export function createSemanticJudgeBatches(
+  candidates: readonly SemanticJudgeCandidate[],
+  serializedBytes: (candidates: readonly SemanticJudgeCandidate[]) => number,
+): SemanticJudgeBatch[] {
+  const batches: SemanticJudgeBatch[] = [];
+  let startIndex = 0;
+  let current: SemanticJudgeCandidate[] = [];
+  for (const [index, candidate] of candidates.entries()) {
+    const proposed = [...current, candidate];
+    const exceedsCount = proposed.length > MAX_SEMANTIC_JUDGE_BATCH_CANDIDATES;
+    const exceedsBytes = serializedBytes(proposed) > MAX_SEMANTIC_JUDGE_REQUEST_BYTES;
+    if (current.length > 0 && (exceedsCount || exceedsBytes)) {
+      batches.push({ startIndex, candidates: current });
+      startIndex = index;
+      current = [candidate];
+    } else {
+      current = proposed;
+    }
+  }
+  if (current.length > 0) batches.push({ startIndex, candidates: current });
+  return batches;
+}
+
+function combineBatchExecutions(
+  executions: readonly SemanticJudgeBatchExecution[],
+): SemanticJudgeBatchExecution {
+  return {
+    successes: executions.flatMap((execution) => execution.successes),
+    failures: executions.flatMap((execution) => execution.failures),
+    attempts: executions.reduce((total, execution) => total + execution.attempts, 0),
+    splits: executions.reduce((total, execution) => total + execution.splits, 0),
+  };
+}
+
+async function settledBatchExecutions(
+  operations: readonly Promise<SemanticJudgeBatchExecution>[],
+): Promise<SemanticJudgeBatchExecution[]> {
+  const settled = await Promise.allSettled(operations);
+  const executions: SemanticJudgeBatchExecution[] = [];
+  let hasRejection = false;
+  let rejection: unknown;
+  for (const result of settled) {
+    if (result.status === "rejected") {
+      if (!hasRejection) rejection = result.reason;
+      hasRejection = true;
+    } else executions.push(result.value);
+  }
+  if (hasRejection) throw rejection;
+  return executions;
+}
+
+async function judgeBatchWithSplit(
+  runner: SemanticJudgeRunner,
+  query: string,
+  batch: SemanticJudgeBatch,
+  signal?: AbortSignal,
+): Promise<SemanticJudgeBatchExecution> {
+  try {
+    const result = await runner(query, batch.candidates, signal);
+    return { successes: [{ ...batch, result }], failures: [], attempts: 1, splits: 0 };
+  } catch (error) {
+    if (signal?.aborted) throw error;
+    if (error instanceof SemanticJudgePayloadTooLargeError && batch.candidates.length > 1) {
+      const midpoint = Math.ceil(batch.candidates.length / 2);
+      const left = await judgeBatchWithSplit(
+        runner,
+        query,
+        { startIndex: batch.startIndex, candidates: batch.candidates.slice(0, midpoint) },
+        signal,
+      );
+      const right = await judgeBatchWithSplit(
+        runner,
+        query,
+        {
+          startIndex: batch.startIndex + midpoint,
+          candidates: batch.candidates.slice(midpoint),
+        },
+        signal,
+      );
+      const combined = combineBatchExecutions([left, right]);
+      return { ...combined, attempts: combined.attempts + 1, splits: combined.splits + 1 };
+    }
+    return { successes: [], failures: [{ ...batch, error }], attempts: 1, splits: 0 };
+  }
+}
+
+export async function judgeSemanticCandidateBatches(
+  runner: SemanticJudgeRunner,
+  query: string,
+  batches: readonly SemanticJudgeBatch[],
+  signal?: AbortSignal,
+): Promise<SemanticJudgeBatchExecution> {
+  return combineBatchExecutions(
+    await settledBatchExecutions(
+      batches.map((batch) => judgeBatchWithSplit(runner, query, batch, signal)),
+    ),
+  );
+}