sift-light 1.0.0

package/src/source-pages.ts

@@ -0,0 +1,112 @@
+import { SiftLightError } from "./errors.js";
+import { SourceDocument, type ByteRange, type SourcePosition } from "./source-document.js";
+
+export interface SourceFragment extends ByteRange {
+  text: string;
+  startPosition: SourcePosition;
+  endPosition: SourcePosition;
+}
+
+export interface SourcePage {
+  fragment: SourceFragment;
+  remaining: ByteRange[];
+  text: string;
+}
+
+/** Half-open ranges, combined before allocating the shared response budget. */
+export function mergeByteRanges(ranges: readonly ByteRange[]): ByteRange[] {
+  const merged: ByteRange[] = [];
+  for (const range of ranges.toSorted((a, b) => a.start - b.start || a.end - b.end)) {
+    if (
+      !Number.isSafeInteger(range.start) ||
+      !Number.isSafeInteger(range.end) ||
+      range.start < 0 ||
+      range.end < range.start
+    ) {
+      throw new SiftLightError("Invalid source range");
+    }
+    const previous = merged.at(-1);
+    if (previous && range.start <= previous.end) previous.end = Math.max(previous.end, range.end);
+    else merged.push({ start: range.start, end: range.end });
+  }
+  return merged;
+}
+
+export function subtractByteRange(ranges: readonly ByteRange[], returned: ByteRange): ByteRange[] {
+  const remaining: ByteRange[] = [];
+  for (const range of ranges) {
+    if (returned.end <= range.start || returned.start >= range.end) {
+      remaining.push({ start: range.start, end: range.end });
+      continue;
+    }
+    if (range.start < returned.start) remaining.push({ start: range.start, end: returned.start });
+    if (returned.end < range.end) remaining.push({ start: returned.end, end: range.end });
+  }
+  return remaining;
+}
+
+function utf8Boundary(document: SourceDocument, offset: number, direction: -1 | 1): number {
+  let byte = Math.max(0, Math.min(document.bytes.length, offset));
+  while (byte > 0 && byte < document.bytes.length) {
+    const value = document.bytes[byte];
+    if (value === undefined || (value & 0xc0) !== 0x80) break;
+    byte += direction;
+  }
+  return byte;
+}
+
+export function renderSourceFragment(fragment: SourceFragment): string {
+  const header = `[source bytes ${String(fragment.start)}..${String(fragment.end)}; ${String(fragment.startPosition.line)}:${String(fragment.startPosition.column)}–${String(fragment.endPosition.line)}:${String(fragment.endPosition.column)}; UTF-8, end exclusive]`;
+  const lines = fragment.text.split("\n");
+  return [
+    header,
+    ...lines.map((text, index) => `${String(fragment.startPosition.line + index)}: ${text}`),
+  ].join("\n");
+}
+
+/** Keep actual source characters; only the explicitly described range is paged. */
+export function sourcePage(
+  document: SourceDocument,
+  ranges: readonly ByteRange[],
+  maxBytes: number,
+  focus?: number,
+): SourcePage {
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 256) {
+    throw new SiftLightError("Source page budget must allow at least 256 bytes");
+  }
+  const gaps = mergeByteRanges(ranges);
+  const range =
+    gaps.find((item) => focus !== undefined && item.start <= focus && focus < item.end) ?? gaps[0];
+  if (!range) throw new SiftLightError("Source range is already complete");
+  document.checkRange(range);
+  document.toCharacterOffset(range.start);
+  document.toCharacterOffset(range.end);
+  const target = Math.max(range.start, Math.min(range.end, focus ?? range.start));
+  let available = Math.max(4, maxBytes - 160);
+  for (;;) {
+    let start = range.start;
+    if (range.end - range.start > available && target - range.start > available / 2) {
+      start = utf8Boundary(document, Math.floor(target - available / 2), 1);
+    }
+    start = Math.max(range.start, start);
+    let end = utf8Boundary(document, Math.min(range.end, start + available), -1);
+    if (end <= start && range.end > range.start) end = utf8Boundary(document, start + 1, 1);
+    const fragment: SourceFragment = {
+      start,
+      end,
+      text: document.slice({ start, end }),
+      startPosition: document.positionAt(start),
+      endPosition: document.positionAt(end),
+    };
+    const text = renderSourceFragment(fragment);
+    if (Buffer.byteLength(text) <= maxBytes) {
+      return {
+        fragment,
+        remaining: subtractByteRange(gaps, fragment).filter((gap) => gap.start < gap.end),
+        text,
+      };
+    }
+    if (available <= 4) throw new SiftLightError("Source metadata exceeds the page budget");
+    available = Math.max(4, Math.floor(available * 0.75));
+  }
+}

package/src/source.ts

@@ -0,0 +1,286 @@
+import { readFile, realpath, stat } from "node:fs/promises";
+import type { Stats } from "node:fs";
+import { abortError, SiftLightError } from "./errors.js";
+import { excerptText } from "./excerpt.js";
+import { isPathInsideCwd, SearchPathPolicy } from "./path-policy.js";
+import {
+  MAX_RESULT_BYTES,
+  MAX_SOURCE_FILE_BYTES,
+  type MatchOccurrence,
+  type SourceRevision,
+} from "./types.js";
+
+export { isPathInsideCwd } from "./path-policy.js";
+
+const SOURCE_RANGE_METADATA_RESERVE_BYTES = 1024;
+const MAX_SOURCE_RANGE_BYTES = MAX_RESULT_BYTES - SOURCE_RANGE_METADATA_RESERVE_BYTES;
+
+export async function getSourceRevision(
+  path: string,
+  onError?: (error: unknown) => void,
+): Promise<SourceRevision | undefined> {
+  try {
+    const metadata = await stat(path);
+    return sourceRevisionFromStats(metadata);
+  } catch (error) {
+    onError?.(error);
+    return undefined;
+  }
+}
+
+export function sourceRevisionFromStats(metadata: Stats): SourceRevision {
+  return {
+    size: metadata.size,
+    mtimeMs: metadata.mtimeMs,
+    ctimeMs: metadata.ctimeMs,
+    ...(metadata.ino !== 0 ? { inode: metadata.ino } : {}),
+    ...(metadata.dev !== 0 ? { device: metadata.dev } : {}),
+  };
+}
+
+export function sameSourceRevision(left: SourceRevision, right: SourceRevision): boolean {
+  return (
+    left.size === right.size &&
+    left.mtimeMs === right.mtimeMs &&
+    (left.ctimeMs === undefined || right.ctimeMs === undefined || left.ctimeMs === right.ctimeMs) &&
+    left.inode === right.inode &&
+    left.device === right.device
+  );
+}
+
+/** Worktree modification-time bounds are inclusive at the lower edge and exclusive at the upper edge. */
+export function matchesModificationTime(
+  revision: SourceRevision,
+  modifiedAfterMs?: number,
+  modifiedBeforeMs?: number,
+): boolean {
+  return (
+    (modifiedAfterMs === undefined || revision.mtimeMs >= modifiedAfterMs) &&
+    (modifiedBeforeMs === undefined || revision.mtimeMs < modifiedBeforeMs)
+  );
+}
+
+function modificationTimeDisplay(value: number): string {
+  const date = new Date(value);
+  return Number.isNaN(date.getTime())
+    ? `${String(value)} Unix ms`
+    : JSON.stringify(date.toISOString());
+}
+
+export function modificationTimeBoundsText(
+  modifiedAfterMs?: number,
+  modifiedBeforeMs?: number,
+): string {
+  if (modifiedAfterMs === undefined && modifiedBeforeMs === undefined) return "";
+  const bounds = [
+    modifiedAfterMs === undefined
+      ? undefined
+      : `mtime >= ${modificationTimeDisplay(modifiedAfterMs)}`,
+    modifiedBeforeMs === undefined
+      ? undefined
+      : `mtime < ${modificationTimeDisplay(modifiedBeforeMs)}`,
+  ].filter((value): value is string => value !== undefined);
+  return ` [Modification-time filter: ${bounds.join("; ")}.]`;
+}
+
+export async function assertExistingSearchPath(path: string, cwd: string): Promise<void> {
+  await new SearchPathPolicy(cwd).assertExistingPath(path);
+}
+
+export async function assertExistingPathInsideCwd(path: string, cwd: string): Promise<void> {
+  if (!isPathInsideCwd(path, cwd)) {
+    throw new SiftLightError("Path must stay within the working directory");
+  }
+  const canonical = await new SearchPathPolicy(cwd).resolveExistingPath(path);
+  if (canonical && !isPathInsideCwd(canonical, await realpath(cwd))) {
+    throw new SiftLightError("Path must stay within the working directory");
+  }
+}
+
+export class SourceTooLargeError extends SiftLightError {
+  constructor(message: string) {
+    super(message);
+    this.name = "SourceTooLargeError";
+  }
+}
+
+export class SourceBudgetTooSmallError extends SiftLightError {
+  constructor() {
+    super("Source target line exceeds the available byte budget");
+    this.name = "SourceBudgetTooSmallError";
+  }
+}
+
+export class SourceLineUnavailableError extends SiftLightError {
+  constructor(line: number) {
+    super(`Source line ${String(line)} is beyond the end of the file`);
+    this.name = "SourceLineUnavailableError";
+  }
+}
+
+export interface SourceRangeRead {
+  text: string;
+  lines: SourceExcerptLine[];
+  startLine: number;
+  endLine: number;
+  truncated: boolean;
+  omittedBefore: number;
+  omittedAfter: number;
+  truncatedLines: number[];
+}
+
+export interface SourceExcerptLine {
+  line: number;
+  text: string;
+  truncated: boolean;
+}
+
+export interface SourceRangeOptions {
+  maxBytes?: number;
+  focus?: MatchOccurrence;
+}
+
+interface SelectedSourceWindow {
+  lines: SourceExcerptLine[];
+  startIndex: number;
+  endIndex: number;
+}
+
+function sourceLineBytes(line: SourceExcerptLine): number {
+  return Buffer.byteLength(`${String(line.line)}: ${line.text}`, "utf8");
+}
+
+function selectSourceWindow(
+  rendered: SourceExcerptLine[],
+  targetIndex: number,
+  maxBytes: number,
+): SelectedSourceWindow {
+  let startIndex = targetIndex;
+  let endIndex = targetIndex;
+  const target = rendered[targetIndex];
+  if (!target) throw new Error("Source target line is unavailable");
+  let bytes = sourceLineBytes(target);
+  if (bytes > maxBytes) throw new SourceBudgetTooSmallError();
+  let canGrowBefore = true;
+  let canGrowAfter = true;
+
+  while (canGrowBefore || canGrowAfter) {
+    let grew = false;
+    if (canGrowBefore) {
+      const candidate = rendered[startIndex - 1];
+      if (candidate === undefined) {
+        canGrowBefore = false;
+      } else if (bytes + 1 + sourceLineBytes(candidate) <= maxBytes) {
+        startIndex -= 1;
+        bytes += 1 + sourceLineBytes(candidate);
+        grew = true;
+      } else {
+        canGrowBefore = false;
+      }
+    }
+
+    if (canGrowAfter) {
+      const candidate = rendered[endIndex + 1];
+      if (candidate === undefined) {
+        canGrowAfter = false;
+      } else if (bytes + 1 + sourceLineBytes(candidate) <= maxBytes) {
+        endIndex += 1;
+        bytes += 1 + sourceLineBytes(candidate);
+        grew = true;
+      } else {
+        canGrowAfter = false;
+      }
+    }
+
+    if (!grew && !canGrowBefore && !canGrowAfter) break;
+  }
+
+  return { lines: rendered.slice(startIndex, endIndex + 1), startIndex, endIndex };
+}
+
+export async function readSourceRange(
+  absolutePath: string,
+  startLine: number,
+  endLine: number,
+  signal?: AbortSignal,
+  targetLine = startLine,
+  options: SourceRangeOptions = {},
+): Promise<SourceRangeRead> {
+  if (signal?.aborted) throw abortError();
+  const maxBytes = options.maxBytes ?? MAX_SOURCE_RANGE_BYTES;
+  if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || maxBytes > MAX_SOURCE_RANGE_BYTES) {
+    throw new Error("Source range byte budget must be within the result body limit");
+  }
+  const metadata = await stat(absolutePath, { bigint: false });
+  if (metadata.size > MAX_SOURCE_FILE_BYTES) {
+    throw new SourceTooLargeError(
+      `Source file exceeds the ${String(MAX_SOURCE_FILE_BYTES)}-byte source limit`,
+    );
+  }
+  const content = await readFile(absolutePath, { signal });
+  if (content.byteLength > MAX_SOURCE_FILE_BYTES) {
+    throw new SourceTooLargeError(
+      `Source file exceeds the ${String(MAX_SOURCE_FILE_BYTES)}-byte source limit`,
+    );
+  }
+  return sourceRangeFromBytes(content, startLine, endLine, targetLine, options);
+}
+
+export function sourceRangeFromBytes(
+  content: Buffer,
+  startLine: number,
+  endLine: number,
+  targetLine = startLine,
+  options: SourceRangeOptions = {},
+): SourceRangeRead {
+  const maxBytes = options.maxBytes ?? MAX_SOURCE_RANGE_BYTES;
+  if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || maxBytes > MAX_SOURCE_RANGE_BYTES)
+    throw new Error("Source range byte budget must be within the result body limit");
+  // ripgrep counts LF-delimited lines. Keep original bytes until byte-based
+  // occurrence offsets have been decoded; re-encoding replacement characters
+  // would move the focus in files containing invalid UTF-8.
+  const lines: Buffer[] = [];
+  let lineStart = 0;
+  for (let newline = content.indexOf(10); newline >= 0; newline = content.indexOf(10, lineStart)) {
+    lines.push(content.subarray(lineStart, newline));
+    lineStart = newline + 1;
+  }
+  lines.push(content.subarray(lineStart));
+  const boundedStart = Math.max(1, startLine);
+  if (boundedStart > lines.length) {
+    throw new SourceLineUnavailableError(targetLine);
+  }
+  const boundedEnd = Math.min(lines.length, Math.max(boundedStart, endLine));
+  if (targetLine < 1 || targetLine > lines.length) {
+    throw new SourceLineUnavailableError(targetLine);
+  }
+  const boundedTarget = Math.min(boundedEnd, Math.max(boundedStart, targetLine));
+  const rendered = Array.from({ length: boundedEnd - boundedStart + 1 }, (_, index) => {
+    const lineNumber = boundedStart + index;
+    const raw = lines[lineNumber - 1];
+    if (!raw) throw new Error("Source line is unavailable");
+    const focus = lineNumber === targetLine ? options.focus : undefined;
+    const start = focus?.range.start.character ?? 0;
+    const end = focus?.range.end.character ?? start;
+    const bytes = focus?.range.encoding === "utf-8" ? raw : undefined;
+    const excerpt = excerptText(
+      raw.toString("utf8").replaceAll("\r", ""),
+      bytes ? bytes.subarray(0, start).toString("utf8").replaceAll("\r", "").length : start,
+      bytes ? bytes.subarray(0, end).toString("utf8").replaceAll("\r", "").length : end,
+    );
+    return { line: lineNumber, text: excerpt.text, truncated: excerpt.truncated };
+  });
+  const selected = selectSourceWindow(rendered, boundedTarget - boundedStart, maxBytes);
+  const omittedBefore = selected.startIndex;
+  const omittedAfter = rendered.length - selected.endIndex - 1;
+  return {
+    text: selected.lines.map((line) => `${String(line.line)}: ${line.text}`).join("\n"),
+    lines: selected.lines,
+    startLine: boundedStart + selected.startIndex,
+    endLine: boundedStart + selected.endIndex,
+    truncated: omittedBefore > 0 || omittedAfter > 0,
+    omittedBefore,
+    omittedAfter,
+    truncatedLines: selected.lines.filter((line) => line.truncated).map((line) => line.line),
+  };
+}

package/src/structural-search.ts

@@ -0,0 +1,125 @@
+import { rangeEvidence } from "./analysis-evidence.js";
+import type { AnalysisResultSet } from "./analysis-types.js";
+import { MAX_ANALYSIS_RESULTS, MAX_ANALYSIS_STORAGE_BYTES } from "./analysis-limits.js";
+import { SiftLightError } from "./errors.js";
+import type { SiftLightInput } from "./service.js";
+import { SourceAccess, SourceBudgetError } from "./source-access.js";
+import { SourceDocumentError } from "./source-document.js";
+import { syntaxLanguage } from "./syntax.js";
+import { normalizeRequest } from "./request.js";
+import { listWorkspaceFiles } from "./workspace-files.js";
+
+export async function structuralSearch(
+  input: SiftLightInput,
+  access: SourceAccess,
+): Promise<AnalysisResultSet> {
+  if (
+    !input.pattern?.trim() ||
+    Buffer.byteLength(input.pattern) > 4_096 ||
+    !input.pattern.isWellFormed()
+  )
+    throw new SiftLightError(
+      "Structural pattern must be nonempty, well-formed and at most 4 KiB; ast-grep $NAME/$$$ARGS metavariables are supported",
+    );
+  const request = normalizeRequest({ ...input, pattern: "" });
+  const files = await listWorkspaceFiles(access.cwd, access.signal, {
+    ...(request.path ? { path: request.path } : {}),
+    glob: request.glob,
+    exclude: request.exclude,
+    hidden: request.hidden,
+  });
+  const result: AnalysisResultSet = {
+    kind: "structure",
+    unit: "occurrences",
+    items: [],
+    partial: files.partial,
+    reasons: [...files.reasons],
+    redact: input.redact ?? false,
+  };
+  let retainedBytes = 0;
+  const supported = files.paths.filter((path) => syntaxLanguage(path));
+  if (files.paths.length && !supported.length)
+    throw new SiftLightError(
+      "Structural patterns require admitted JS/TS/TSX/Go source; no supported source files were found",
+    );
+  result.stats = {
+    filesEnumerated: files.paths.length,
+    filesSkipped: files.paths.length - supported.length,
+  };
+  for (const path of files.paths) {
+    if (!syntaxLanguage(path)) continue;
+    try {
+      // oxlint-disable-next-line no-await-in-loop -- sequential parser admission shares the source and output budgets.
+      const document = await access.load(path);
+      // oxlint-disable-next-line no-await-in-loop -- parser execution is owned by the serialized syntax queue.
+      const syntax = await access.pattern(document, input.pattern);
+      if (syntax.status !== "ok") {
+        result.partial = true;
+        result.reasons.push(`${path}: syntax ${syntax.status}; structural matches withheld`);
+        continue;
+      }
+      for (const match of syntax.patternMatches ?? []) {
+        const range = {
+          start: document.toByteOffset(match.start),
+          end: document.toByteOffset(match.end),
+        };
+        const evidence = rangeEvidence(document, range);
+        const item = {
+          path: document.path,
+          line: document.lineAt(range.start),
+          source: document.reference,
+          range,
+          label: "AST pattern match",
+          excerpt: evidence.excerpt,
+          details: {
+            kind: "structural-match",
+            certainty: "syntax",
+            score: 90,
+            rankingReason: "AST structure and repeated metavariable equality",
+            excerptRange: evidence.excerptRange,
+            excerptTruncated: evidence.excerptTruncated,
+          },
+        };
+        const bytes = Buffer.byteLength(JSON.stringify(item));
+        if (
+          result.items.length >= MAX_ANALYSIS_RESULTS ||
+          retainedBytes + bytes > MAX_ANALYSIS_STORAGE_BYTES - 65_536
+        ) {
+          result.partial = true;
+          result.reasons.push("Structural evidence storage limit reached: 50,000 items / 32 MiB");
+          break;
+        }
+        result.items.push(item);
+        retainedBytes += bytes;
+      }
+      if (
+        result.items.length >= MAX_ANALYSIS_RESULTS ||
+        result.reasons.at(-1)?.startsWith("Structural evidence storage")
+      )
+        break;
+    } catch (error) {
+      if (error instanceof SourceBudgetError) {
+        result.partial = true;
+        result.reasons.push(error.message);
+        break;
+      }
+      if (!(error instanceof SourceDocumentError)) throw error;
+      result.partial = true;
+      result.reasons.push(`${path}: ${error.message}`);
+    }
+  }
+  result.filesRead = access.filesRead;
+  result.bytesRead = access.bytesRead;
+  result.coverage = { astPatterns: result.partial ? "partial" : "complete" };
+  result.scope = {
+    path: request.path ?? ".",
+    requestedPath: request.path ?? ".",
+    glob: request.glob,
+    exclude: request.exclude,
+    hidden: request.hidden,
+    ignorePolicy: request.ignorePolicy ?? "respect",
+    expandedToProjectRoot: false,
+    assertion: request.path && request.path !== "." ? "requested-scope" : "project-wide",
+  };
+  return result;
+}