sift-light 1.0.0

package/src/session-summary.ts

@@ -0,0 +1,100 @@
+import type { SiftLightLocale } from "./config.js";
+import { SIFT_LIGHT_VERSION } from "./package-version.js";
+import type { SiftLightInput } from "./service.js";
+import type { SiftLightResult } from "./types.js";
+
+export const SESSION_STATUS_KEY = "sift-light_session";
+
+export interface SessionSummarySnapshot {
+  queries: number;
+  completeQueries: number;
+  partialQueries: number;
+  organizedQueries: number;
+  failedCalls: number;
+}
+
+function isNewQuery(input: SiftLightInput): boolean {
+  return (
+    input.cursor === undefined &&
+    input.sourceCursor === undefined &&
+    input.operationId === undefined
+  );
+}
+
+function wasAutomaticallyOrganized(input: SiftLightInput, result: SiftLightResult): boolean {
+  const autoMode = input.mode === undefined || input.mode === "auto";
+  return autoMode && input.limit === undefined && result.details.summaryFilesShown !== undefined;
+}
+
+function formatChineseCompleteness(snapshot: SessionSummarySnapshot): string {
+  const { completeQueries, partialQueries, queries } = snapshot;
+  if (queries === 0) return "暂无成功查询";
+  const unfinishedQueries = queries - completeQueries - partialQueries;
+  if (completeQueries > 0 && partialQueries === 0 && unfinishedQueries === 0) return "结果全部完整";
+  const parts: string[] = [];
+  if (completeQueries > 0) parts.push(`${String(completeQueries)} 次结果完整`);
+  if (partialQueries > 0) parts.push(`${String(partialQueries)} 次仅获得部分结果并已明确标注`);
+  if (unfinishedQueries > 0) parts.push(`${String(unfinishedQueries)} 次结果尚未完成`);
+  return parts.join("；");
+}
+
+function formatEnglishCompleteness(snapshot: SessionSummarySnapshot): string {
+  const { completeQueries, partialQueries, queries } = snapshot;
+  if (queries === 0) return "no successful queries";
+  const unfinishedQueries = queries - completeQueries - partialQueries;
+  if (completeQueries > 0 && partialQueries === 0 && unfinishedQueries === 0)
+    return "all results complete";
+  const parts: string[] = [];
+  if (completeQueries > 0) parts.push(`${String(completeQueries)} complete`);
+  if (partialQueries > 0) parts.push(`${String(partialQueries)} partial and clearly marked`);
+  if (unfinishedQueries > 0) parts.push(`${String(unfinishedQueries)} not yet complete`);
+  return parts.join("; ");
+}
+export class SessionSummary {
+  #snapshot: SessionSummarySnapshot = {
+    queries: 0,
+    completeQueries: 0,
+    partialQueries: 0,
+    organizedQueries: 0,
+    failedCalls: 0,
+  };
+
+  record(input: SiftLightInput, result: SiftLightResult): void {
+    if (!isNewQuery(input)) return;
+    this.#snapshot.queries += 1;
+    if (result.details.status === "complete") this.#snapshot.completeQueries += 1;
+    if (result.details.status === "partial") this.#snapshot.partialQueries += 1;
+    if (wasAutomaticallyOrganized(input, result)) this.#snapshot.organizedQueries += 1;
+  }
+
+  recordFailure(): void {
+    this.#snapshot.failedCalls += 1;
+  }
+
+  get snapshot(): SessionSummarySnapshot {
+    return { ...this.#snapshot };
+  }
+
+  format(locale: SiftLightLocale): string | undefined {
+    const { failedCalls, organizedQueries, queries } = this.#snapshot;
+    if (queries === 0 && failedCalls === 0) return undefined;
+    if (locale === "zh-CN") {
+      const organized =
+        organizedQueries > 0 ? `；${String(organizedQueries)} 次结果已自动按文件整理` : "";
+      const failures = failedCalls > 0 ? `；${String(failedCalls)} 次调用失败` : "";
+      return `sift-light ${SIFT_LIGHT_VERSION}：已处理 ${String(queries)} 次查询，${formatChineseCompleteness(this.#snapshot)}${organized}${failures}`;
+    }
+    const queryWord = queries === 1 ? "query" : "queries";
+    let organized = "";
+    if (organizedQueries > 0) {
+      const resultWord = organizedQueries === 1 ? "result" : "results";
+      organized = `; ${organizedQueries} ${resultWord} automatically organized by file`;
+    }
+    let failures = "";
+    if (failedCalls > 0) {
+      const callWord = failedCalls === 1 ? "call" : "calls";
+      failures = `; ${failedCalls} failed ${callWord}`;
+    }
+    return `sift-light ${SIFT_LIGHT_VERSION}: handled ${String(queries)} ${queryWord}; ${formatEnglishCompleteness(this.#snapshot)}${organized}${failures}`;
+  }
+}

package/src/snapshot-store.ts

@@ -0,0 +1,203 @@
+import { randomUUID } from "node:crypto";
+import { CursorError, SiftLightError } from "./errors.js";
+import { MAX_SEARCH_STORAGE_BYTES, MAX_STORED_OCCURRENCES } from "./types.js";
+import type { SearchScan, SearchSnapshot } from "./types.js";
+
+export type CursorKind = "matches" | "summary";
+
+export interface SnapshotStoreOptions {
+  ttlMs?: number;
+  maxSnapshots?: number;
+  maxTotalStoredMatches?: number;
+  maxTotalStoredBytes?: number;
+  maxTotalStoredOccurrences?: number;
+  now?: () => number;
+}
+
+export interface ResolvedCursor {
+  snapshot: SearchSnapshot;
+  offset: number;
+  kind: CursorKind;
+  selectionKey: string;
+}
+
+export class SnapshotStore {
+  readonly #snapshots = new Map<string, SearchSnapshot>();
+  readonly #expired = new Set<string>();
+  readonly #ttlMs: number;
+  readonly #maxSnapshots: number;
+  readonly #maxTotalStoredMatches: number;
+  readonly #maxTotalStoredBytes: number;
+  readonly #maxTotalStoredOccurrences: number;
+  readonly #now: () => number;
+
+  constructor(options: SnapshotStoreOptions = {}) {
+    this.#ttlMs = options.ttlMs ?? 10 * 60 * 1000;
+    this.#maxSnapshots = options.maxSnapshots ?? 20;
+    this.#maxTotalStoredMatches = options.maxTotalStoredMatches ?? 100_000;
+    this.#maxTotalStoredBytes = options.maxTotalStoredBytes ?? MAX_SEARCH_STORAGE_BYTES;
+    this.#maxTotalStoredOccurrences = options.maxTotalStoredOccurrences ?? MAX_STORED_OCCURRENCES;
+    this.#now = options.now ?? Date.now;
+  }
+
+  create(scan: SearchScan): SearchSnapshot {
+    this.sweep();
+    if (
+      this.#scanBytes(scan) > this.#maxTotalStoredBytes ||
+      this.#scanOccurrences(scan) > this.#maxTotalStoredOccurrences
+    )
+      throw new SiftLightError("Search snapshot exceeds the session storage budget");
+    const now = this.#now();
+    const snapshot: SearchSnapshot = {
+      ...scan,
+      id: randomUUID(),
+      createdAt: now,
+      lastAccessedAt: now,
+    };
+    this.#snapshots.set(snapshot.id, snapshot);
+    this.#evictToBounds();
+    return snapshot;
+  }
+
+  cursor(
+    snapshot: SearchSnapshot,
+    offset: number,
+    kind: CursorKind = "matches",
+    selectionKey = "all",
+  ): string {
+    if (!Number.isSafeInteger(offset) || offset < 0) {
+      throw new CursorError(
+        "Cannot create a cursor with an invalid offset",
+        "E_CURSOR_OFFSET_INVALID",
+      );
+    }
+    if (!/^(?:all|[0-9a-f]{16})$/.test(selectionKey)) {
+      throw new CursorError("Cannot create a cursor with an invalid selection key");
+    }
+    return `${snapshot.id}.${kind}.${offset.toString(36)}.${selectionKey}`;
+  }
+
+  resolve(cursor: string): ResolvedCursor {
+    this.sweep();
+    const parts = cursor.match(/^(.+)\.(matches|summary)\.([0-9a-z]+)\.(all|[0-9a-f]{16})$/);
+    if (!parts) {
+      throw new CursorError(
+        "Invalid cursor. Start a new search to obtain a fresh cursor.",
+        "E_CURSOR_MALFORMED",
+      );
+    }
+    const [, id, rawKind, rawOffset, selectionKey] = parts;
+    if (!id || !rawKind || !rawOffset || !selectionKey) {
+      throw new CursorError(
+        "Invalid cursor. Start a new search to obtain a fresh cursor.",
+        "E_CURSOR_MALFORMED",
+      );
+    }
+    const kind: CursorKind = rawKind === "summary" ? "summary" : "matches";
+
+    const offset = Number.parseInt(rawOffset, 36);
+    const snapshot = this.#snapshots.get(id);
+    if (!snapshot)
+      throw new CursorError(
+        this.#expired.has(id)
+          ? "Cursor expired or was evicted. Run the search again."
+          : "Cursor was not found. Run the search again.",
+        this.#expired.has(id) ? "E_CURSOR_EXPIRED" : "E_CURSOR_NOT_FOUND",
+      );
+    const maximumOffset = kind === "summary" ? snapshot.fileCounts.size : snapshot.matches.length;
+    if (!Number.isSafeInteger(offset) || offset < 0 || offset > maximumOffset) {
+      throw new CursorError(
+        "Cursor offset is outside the retained search snapshot.",
+        "E_CURSOR_OFFSET_INVALID",
+      );
+    }
+
+    snapshot.lastAccessedAt = this.#now();
+    return { snapshot, offset, kind, selectionKey };
+  }
+
+  delete(snapshot: SearchSnapshot): boolean {
+    return this.#snapshots.delete(snapshot.id);
+  }
+
+  clear(): void {
+    for (const id of this.#snapshots.keys()) this.#rememberExpired(id);
+    this.#snapshots.clear();
+  }
+
+  sweep(): void {
+    const cutoff = this.#now() - this.#ttlMs;
+    for (const [id, snapshot] of this.#snapshots) {
+      if (snapshot.lastAccessedAt < cutoff) {
+        this.#snapshots.delete(id);
+        this.#rememberExpired(id);
+      }
+    }
+  }
+
+  get size(): number {
+    this.sweep();
+    return this.#snapshots.size;
+  }
+
+  get storedMatches(): number {
+    this.sweep();
+    return this.#totalStoredMatches();
+  }
+
+  #evictToBounds(): void {
+    while (
+      this.#snapshots.size > this.#maxSnapshots ||
+      this.#totalStoredMatches() > this.#maxTotalStoredMatches ||
+      [...this.#snapshots.values()].reduce((total, scan) => total + this.#scanBytes(scan), 0) >
+        this.#maxTotalStoredBytes ||
+      [...this.#snapshots.values()].reduce(
+        (total, scan) => total + this.#scanOccurrences(scan),
+        0,
+      ) > this.#maxTotalStoredOccurrences
+    ) {
+      let oldest: SearchSnapshot | undefined;
+      for (const snapshot of this.#snapshots.values()) {
+        if (!oldest || snapshot.lastAccessedAt < oldest.lastAccessedAt) oldest = snapshot;
+      }
+      if (!oldest) break;
+      this.#snapshots.delete(oldest.id);
+      this.#rememberExpired(oldest.id);
+    }
+  }
+
+  #totalStoredMatches(): number {
+    let total = 0;
+    for (const snapshot of this.#snapshots.values()) total += snapshot.matches.length;
+    return total;
+  }
+
+  #scanBytes(scan: SearchScan): number {
+    return (
+      scan.retention?.accountedBytes ??
+      Buffer.byteLength(
+        JSON.stringify({
+          matches: scan.matches,
+          fileCounts: [...scan.fileCounts],
+          sourceRevisions: [...scan.sourceRevisions],
+        }),
+      )
+    );
+  }
+
+  #scanOccurrences(scan: SearchScan): number {
+    return (
+      scan.retention?.retainedOccurrences ??
+      scan.matches.reduce((total, match) => total + match.occurrences.length, 0)
+    );
+  }
+
+  #rememberExpired(id: string): void {
+    this.#expired.add(id);
+    while (this.#expired.size > this.#maxSnapshots * 4) {
+      const oldest = this.#expired.values().next().value;
+      if (oldest === undefined) break;
+      this.#expired.delete(oldest);
+    }
+  }
+}

package/src/source-access.ts

@@ -0,0 +1,288 @@
+import { extname, resolve } from "node:path";
+import {
+  MAX_STRUCTURE_BYTES,
+  MAX_STRUCTURE_FILES,
+  MAX_SYNTAX_CACHE_ENTRIES,
+  MAX_SYNTAX_CACHE_NODES,
+} from "./analysis-limits.js";
+import { abortError, SiftLightError } from "./errors.js";
+import { readGitSource } from "./git-source.js";
+import {
+  readWorkspaceDocument,
+  SourceDocument,
+  SourceDocumentError,
+  type SourceReference,
+} from "./source-document.js";
+import { parseSyntax, type SyntaxAnalysis } from "./syntax.js";
+import { getSourceRevision } from "./source.js";
+
+function noop(): void {}
+
+interface CachedSyntax {
+  analysis: SyntaxAnalysis;
+  nodes: number;
+}
+
+interface SyntaxParseResult {
+  analysis: SyntaxAnalysis;
+  cacheHit: boolean;
+}
+
+interface SourceBudgetLedger {
+  readBytes: number;
+  verificationBytes: number;
+}
+
+/** One parser owner across calls, with a bounded content-addressed syntax cache. */
+export class SyntaxQueue {
+  #tail: Promise<void> = Promise.resolve();
+  #generation = new AbortController();
+  readonly #cache = new Map<string, CachedSyntax>();
+  #cachedNodes = 0;
+
+  async parse(document: SourceDocument, signal?: AbortSignal): Promise<SyntaxAnalysis> {
+    return (await this.parseWithMetrics(document, signal)).analysis;
+  }
+
+  async parseWithMetrics(
+    document: SourceDocument,
+    signal?: AbortSignal,
+    pattern?: string,
+  ): Promise<SyntaxParseResult> {
+    const combined = signal
+      ? AbortSignal.any([signal, this.#generation.signal])
+      : this.#generation.signal;
+    if (combined.aborted) throw abortError();
+    const predecessor = this.#tail;
+    let release: () => void = noop;
+    this.#tail = new Promise<void>((done) => {
+      release = done;
+    });
+    try {
+      await predecessor;
+      if (combined.aborted) throw abortError();
+      if (!document.utf8)
+        throw new SourceDocumentError("encoding", "Syntax requires lossless UTF-8 source");
+      const origin = document.reference.origin;
+      const revision = origin.kind === "worktree" ? origin.contentHash : origin.blob;
+      const key = `${extname(document.path).toLowerCase()}\0${revision}\0${pattern ?? ""}`;
+      const cached = this.#cache.get(key);
+      if (cached) {
+        this.#cache.delete(key);
+        this.#cache.set(key, cached);
+        return { analysis: cached.analysis, cacheHit: true };
+      }
+      const analysis = await parseSyntax(document.path, document.text, combined, pattern);
+      const entry = { analysis, nodes: analysis.nodes.length };
+      this.#cache.set(key, entry);
+      this.#cachedNodes += entry.nodes;
+      while (
+        this.#cache.size > MAX_SYNTAX_CACHE_ENTRIES ||
+        this.#cachedNodes > MAX_SYNTAX_CACHE_NODES
+      ) {
+        const oldest = this.#cache.entries().next().value;
+        if (!oldest) break;
+        this.#cache.delete(oldest[0]);
+        this.#cachedNodes -= oldest[1].nodes;
+      }
+      return { analysis, cacheHit: false };
+    } finally {
+      release();
+    }
+  }
+
+  clear(): void {
+    this.#generation.abort();
+    this.#generation = new AbortController();
+    this.#cache.clear();
+    this.#cachedNodes = 0;
+  }
+
+  async shutdown(): Promise<void> {
+    this.clear();
+    await this.#tail;
+  }
+}
+
+export class SourceBudgetError extends SiftLightError {
+  readonly reason = "structural-read-budget-exhausted";
+}
+
+/** Created for a single tool request, then discarded, including its parse results. */
+export class SourceAccess {
+  readonly cwd: string;
+  readonly signal: AbortSignal | undefined;
+  readonly #queue: SyntaxQueue;
+  readonly #maxFiles: number;
+  readonly #documents = new Map<string, Promise<SourceDocument>>();
+  readonly #syntax = new Map<SourceDocument, Promise<SyntaxAnalysis>>();
+  readonly #maxVerificationBytes: number;
+  readonly #budget: SourceBudgetLedger;
+  #bytes = 0;
+  #syntaxParses = 0;
+  #syntaxCacheHits = 0;
+  #readTail: Promise<void> = Promise.resolve();
+
+  constructor(
+    cwd: string,
+    queue: SyntaxQueue,
+    signal?: AbortSignal,
+    options: {
+      maxFiles?: number;
+      maxVerificationBytes?: number;
+      budget?: SourceBudgetLedger;
+    } = {},
+  ) {
+    this.cwd = cwd;
+    this.#queue = queue;
+    this.signal = signal;
+    this.#maxFiles = options.maxFiles ?? MAX_STRUCTURE_FILES;
+    this.#maxVerificationBytes = options.maxVerificationBytes ?? MAX_STRUCTURE_BYTES;
+    this.#budget = options.budget ?? { readBytes: 0, verificationBytes: 0 };
+  }
+
+  get filesRead(): number {
+    return this.#documents.size;
+  }
+  get bytesRead(): number {
+    return this.#bytes;
+  }
+  get maxFiles(): number {
+    return this.#maxFiles;
+  }
+  get syntaxParses(): number {
+    return this.#syntaxParses;
+  }
+  get syntaxCacheHits(): number {
+    return this.#syntaxCacheHits;
+  }
+
+  /** Derive a fresh request-local access scope that shares only the parser owner and limits. */
+  withSignal(signal: AbortSignal): SourceAccess {
+    return new SourceAccess(this.cwd, this.#queue, signal, {
+      maxFiles: this.#maxFiles,
+      maxVerificationBytes: this.#maxVerificationBytes,
+    });
+  }
+
+  /** Create one request-owned batch with a fresh document cache and the shared byte ledger. */
+  batch(maxFiles: number): SourceAccess {
+    return new SourceAccess(this.cwd, this.#queue, this.signal, {
+      maxFiles,
+      maxVerificationBytes: this.#maxVerificationBytes,
+      budget: this.#budget,
+    });
+  }
+
+  async load(path: string, expected?: SourceReference): Promise<SourceDocument> {
+    if (this.signal?.aborted) throw abortError();
+    if (expected && resolve(this.cwd, expected.path) !== resolve(this.cwd, path)) {
+      throw new SiftLightError("Source reference path does not match the requested file");
+    }
+    const key = JSON.stringify([resolve(this.cwd, path), expected?.origin]);
+    const existing = this.#documents.get(key);
+    if (existing) return existing;
+    if (this.#documents.size >= this.#maxFiles) {
+      throw new SourceBudgetError(
+        `Structural scan reached the ${String(this.#maxFiles)}-file limit`,
+      );
+    }
+    const pending = this.#read(path, expected, false);
+    this.#documents.set(key, pending);
+    return pending;
+  }
+
+  async #read(
+    path: string,
+    expected: SourceReference | undefined,
+    verification: boolean,
+  ): Promise<SourceDocument> {
+    const predecessor = this.#readTail;
+    let release: () => void = noop;
+    this.#readTail = new Promise<void>((done) => {
+      release = done;
+    });
+    try {
+      await predecessor;
+      return await this.#readOnce(path, expected, verification);
+    } finally {
+      release();
+    }
+  }
+
+  async #readOnce(
+    path: string,
+    expected: SourceReference | undefined,
+    verification: boolean,
+  ): Promise<SourceDocument> {
+    let document: SourceDocument;
+    const consumed = verification ? this.#budget.verificationBytes : this.#budget.readBytes;
+    const budget = verification ? this.#maxVerificationBytes : MAX_STRUCTURE_BYTES;
+    const remaining = budget - consumed;
+    if (remaining <= 0)
+      throw new SourceBudgetError("Structural scan reached the 32 MiB read limit");
+    if (expected?.origin.kind !== "git") {
+      const metadata = await getSourceRevision(resolve(this.cwd, path));
+      if (metadata && metadata.size > remaining)
+        throw new SourceBudgetError(
+          "Next source exceeds the remaining 32 MiB structural read budget",
+        );
+    }
+    if (expected?.origin.kind === "git") {
+      const origin = expected.origin;
+      const raw = await readGitSource(
+        this.cwd,
+        { path, commit: origin.commit, blob: origin.blob },
+        this.signal,
+        { maxBytes: remaining },
+      );
+      if (!raw.content || !raw.origin) {
+        throw new SourceDocumentError(
+          "source-unavailable",
+          raw.reason ?? `Git source is ${raw.sourceStatus}`,
+        );
+      }
+      document = new SourceDocument({ path, origin: raw.origin }, raw.content);
+    } else {
+      document = await readWorkspaceDocument(
+        path,
+        this.cwd,
+        this.signal,
+        expected?.origin,
+        remaining,
+      );
+    }
+    this.#bytes += document.bytes.length;
+    if (verification) this.#budget.verificationBytes += document.bytes.length;
+    else this.#budget.readBytes += document.bytes.length;
+    if (!verification && this.#budget.readBytes > MAX_STRUCTURE_BYTES)
+      throw new SourceBudgetError("Structural scan reached the 32 MiB read limit");
+    return document;
+  }
+
+  syntax(document: SourceDocument): Promise<SyntaxAnalysis> {
+    let pending = this.#syntax.get(document);
+    if (!pending) {
+      pending = this.#queue.parseWithMetrics(document, this.signal).then((parsed) => {
+        if (parsed.cacheHit) this.#syntaxCacheHits += 1;
+        else this.#syntaxParses += 1;
+        return parsed.analysis;
+      });
+      this.#syntax.set(document, pending);
+    }
+    return pending;
+  }
+
+  async pattern(document: SourceDocument, pattern: string): Promise<SyntaxAnalysis> {
+    return (await this.#queue.parseWithMetrics(document, this.signal, pattern)).analysis;
+  }
+
+  releaseSyntax(document: SourceDocument): void {
+    this.#syntax.delete(document);
+  }
+
+  /** Saved source validation must reread the expected version, not consult the request cache. */
+  refresh(path: string, expected: SourceReference): Promise<SourceDocument> {
+    return this.#read(path, expected, true);
+  }
+}