sift-light 1.0.0

package/src/structure.ts

@@ -0,0 +1,259 @@
+import { isAbsolute, resolve } from "node:path";
+import { consumeCappedLines } from "./capped-lines.js";
+import { runOwnedProcess } from "./owned-process.js";
+import { abortError, SiftLightError } from "./errors.js";
+import { getSourceRevision, sameSourceRevision } from "./source.js";
+import {
+  MAX_PROTOCOL_LINE_BYTES,
+  MAX_SOURCE_FILE_BYTES,
+  type SourceRevision,
+  type StructureDetails,
+  type StructureSymbol,
+} from "./types.js";
+
+export const CTAGS_CAPABILITY_ARGUMENTS = [
+  "--output-format=json",
+  "--fields=+ne",
+  "--extras=-p",
+] as const;
+
+export interface StructureInspectionRequest {
+  absolutePath: string;
+  cwd: string;
+  line: number;
+  expectedRevision?: SourceRevision;
+}
+
+export interface StructureInspection {
+  details: StructureDetails;
+  currentRevision?: SourceRevision;
+}
+
+export interface CodeStructureProvider {
+  inspect(request: StructureInspectionRequest, signal?: AbortSignal): Promise<StructureInspection>;
+}
+
+export interface CtagsTag {
+  path: string;
+  name: string;
+  language?: string;
+  kind?: string;
+  scope?: string;
+  line?: number;
+  end?: number;
+}
+
+export interface CtagsStructureProviderOptions {
+  executable?: string;
+  maxFileBytes?: number;
+  runCtags?: (absolutePath: string, cwd: string, signal?: AbortSignal) => Promise<CtagsTag[]>;
+}
+
+class CtagsCommandError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "CtagsCommandError";
+  }
+}
+
+class CtagsProtocolError extends SiftLightError {}
+
+function hasCode(error: unknown, code: string): boolean {
+  return error instanceof Error && "code" in error && error.code === code;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function asOptionalString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function asOptionalPositiveInteger(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined;
+}
+
+function parseCtagsTag(value: unknown): CtagsTag | undefined {
+  if (!isRecord(value)) return undefined;
+  const hasTagType = Object.entries(value).some(
+    ([key, entry]) => key === "_type" && entry === "tag",
+  );
+  if (!hasTagType) return undefined;
+  const path = asOptionalString(value.path);
+  const name = asOptionalString(value.name);
+  const language = asOptionalString(value.language);
+  const kind = asOptionalString(value.kind);
+  const scope = asOptionalString(value.scope);
+  const line = asOptionalPositiveInteger(value.line);
+  const end = asOptionalPositiveInteger(value.end);
+  if (!path || !name) return undefined;
+  return {
+    path,
+    name,
+    ...(language ? { language } : {}),
+    ...(kind ? { kind } : {}),
+    ...(scope ? { scope } : {}),
+    ...(line ? { line } : {}),
+    ...(end ? { end } : {}),
+  };
+}
+
+async function runCtagsCommand(
+  executable: string,
+  absolutePath: string,
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<CtagsTag[]> {
+  const tags: CtagsTag[] = [];
+  const { code, stderr } = await runOwnedProcess(
+    {
+      executable,
+      args: [...CTAGS_CAPABILITY_ARGUMENTS, absolutePath],
+      cwd,
+      ...(signal ? { signal } : {}),
+    },
+    async (stdout) => {
+      try {
+        await consumeCappedLines(
+          stdout,
+          (line) => {
+            if (line.length === 0) return;
+            let value: unknown;
+            try {
+              value = JSON.parse(line);
+            } catch (error) {
+              throw new CtagsProtocolError("Failed to parse Universal Ctags JSON output", {
+                cause: error,
+              });
+            }
+            const tag = parseCtagsTag(value);
+            if (tag) tags.push(tag);
+          },
+          { maxLineBytes: MAX_PROTOCOL_LINE_BYTES },
+        );
+      } catch (error) {
+        if (error instanceof Error && error.message.startsWith("Input line exceeds the ")) {
+          throw new CtagsProtocolError(error.message, { cause: error });
+        }
+        throw error;
+      }
+    },
+  );
+  if (code !== 0) {
+    throw new CtagsCommandError(stderr.trim() || `ctags exited with status ${String(code)}`);
+  }
+  return tags;
+}
+
+function pathMatches(tagPath: string, absolutePath: string, cwd: string): boolean {
+  return resolve(isAbsolute(tagPath) ? tagPath : resolve(cwd, tagPath)) === resolve(absolutePath);
+}
+
+function symbolFromTag(tag: CtagsTag): StructureSymbol | undefined {
+  if (tag.line === undefined || tag.end === undefined || tag.end < tag.line) return undefined;
+  return {
+    name: tag.name,
+    kind: tag.kind ?? "unknown",
+    scope: tag.scope ? [tag.scope] : [],
+    range: { startLine: tag.line, endLine: tag.end },
+  };
+}
+
+function chooseEnclosingSymbol(
+  tags: CtagsTag[],
+  absolutePath: string,
+  cwd: string,
+  line: number,
+): StructureSymbol | undefined {
+  const candidates = tags
+    .filter((tag) => pathMatches(tag.path, absolutePath, cwd))
+    .map(symbolFromTag)
+    .filter((symbol): symbol is StructureSymbol => symbol !== undefined)
+    .filter((symbol) => symbol.range.startLine <= line && line <= symbol.range.endLine)
+    .toSorted((left, right) => {
+      const leftSize = left.range.endLine - left.range.startLine;
+      const rightSize = right.range.endLine - right.range.startLine;
+      if (leftSize !== rightSize) return leftSize - rightSize;
+      return right.scope.length - left.scope.length;
+    });
+  return candidates[0];
+}
+
+export function createCtagsStructureProvider(
+  options: CtagsStructureProviderOptions = {},
+): CodeStructureProvider {
+  const executable = options.executable ?? "ctags";
+  const maxFileBytes = options.maxFileBytes ?? MAX_SOURCE_FILE_BYTES;
+  const runCtags =
+    options.runCtags ??
+    ((absolutePath, cwd, signal) => runCtagsCommand(executable, absolutePath, cwd, signal));
+
+  return {
+    async inspect(request, signal) {
+      if (signal?.aborted) throw abortError();
+      const currentRevision = await getSourceRevision(request.absolutePath);
+      if (!currentRevision) {
+        return { details: { status: "source-unavailable", provider: "universal-ctags" } };
+      }
+      if (
+        request.expectedRevision &&
+        !sameSourceRevision(request.expectedRevision, currentRevision)
+      ) {
+        return {
+          details: { status: "source-changed", provider: "universal-ctags" },
+          currentRevision,
+        };
+      }
+      if (currentRevision.size > maxFileBytes) {
+        return {
+          details: { status: "file-too-large", provider: "universal-ctags" },
+          currentRevision,
+        };
+      }
+
+      let tags: CtagsTag[];
+      try {
+        tags = await runCtags(request.absolutePath, request.cwd, signal);
+      } catch (error) {
+        if (signal?.aborted || (error instanceof Error && error.name === "AbortError")) {
+          throw abortError();
+        }
+        if (hasCode(error, "ENOENT") || error instanceof CtagsCommandError) {
+          return {
+            details: {
+              status: "provider-unavailable",
+              provider: "universal-ctags",
+              reason:
+                "Universal Ctags is unavailable; install universal-ctags or use JS/TS/Python outline support",
+            },
+            currentRevision,
+          };
+        }
+        if (error instanceof CtagsProtocolError) {
+          return {
+            details: {
+              status: "parse-error",
+              provider: "universal-ctags",
+              reason: "Universal Ctags returned invalid JSON output; check the installed provider",
+            },
+            currentRevision,
+          };
+        }
+        throw error;
+      }
+
+      const symbol = chooseEnclosingSymbol(tags, request.absolutePath, request.cwd, request.line);
+      const language = tags.find((tag) => tag.language)?.language;
+      return {
+        details: {
+          status: symbol ? "available" : "no-symbol",
+          provider: "universal-ctags",
+          ...(language ? { language } : {}),
+          ...(symbol ? { symbol, range: symbol.range } : {}),
+        },
+        currentRevision,
+      };
+    },
+  };
+}

package/src/summary-previews.ts

@@ -0,0 +1,96 @@
+import { abortError } from "./errors.js";
+import { excerptText } from "./excerpt.js";
+import { readWorkspaceDocument } from "./source-document.js";
+import { sameSourceRevision } from "./source.js";
+import { MAX_SOURCE_FILE_BYTES, type SearchSnapshot } from "./types.js";
+
+function hasBareCarriageReturn(bytes: Uint8Array): boolean {
+  for (let index = 0; index < bytes.length; index += 1)
+    if (bytes[index] === 13 && bytes[index + 1] !== 10) return true;
+  return false;
+}
+
+/** At most five reads, two disjoint windows/file, seven lines/window. */
+export async function summarySourcePreviews(
+  snapshot: SearchSnapshot,
+  paths: string[],
+  maxBytes: number,
+  cwd: string,
+  signal?: AbortSignal,
+) {
+  const rows: string[] = [];
+  const indices: number[] = [];
+  let bytes = 0;
+  let filesRead = 0;
+  let windows = 0;
+  const reasons: string[] = [];
+  for (const path of paths) {
+    if (filesRead >= 5 || maxBytes - bytes < 256) break;
+    const matches = snapshot.matches.flatMap((match, index) =>
+      match.displayPath === path ? [{ match, index }] : [],
+    );
+    const first = matches[0];
+    if (!first) continue;
+    if (matches.some(({ match }) => match.lineContent.includes("\r"))) {
+      reasons.push(`${path}: preview skipped for non-standard line endings`);
+      continue;
+    }
+    if (matches.some(({ match }) => match.lineTruncated || match.occurrences.length > 20)) {
+      reasons.push(`${path}: preview skipped for dense or truncated matching lines`);
+      continue;
+    }
+    const revision = snapshot.sourceRevisions.get(first.match.absolutePath);
+    if (!revision || revision.size > MAX_SOURCE_FILE_BYTES) {
+      reasons.push(`${path}: preview source unverified or over 5 MiB`);
+      continue;
+    }
+    filesRead += 1;
+    try {
+      // oxlint-disable-next-line no-await-in-loop -- prior rendered windows determine whether later source reads fit the shared preview budget.
+      const document = await readWorkspaceDocument(path, cwd, signal);
+      if (
+        document.reference.origin.kind !== "worktree" ||
+        !sameSourceRevision(revision, document.reference.origin.revision) ||
+        !document.utf8
+      ) {
+        reasons.push(`${path}: preview source changed or is not lossless UTF-8`);
+        continue;
+      }
+      if (hasBareCarriageReturn(document.bytes)) {
+        reasons.push(`${path}: preview skipped for non-standard line endings`);
+        continue;
+      }
+      let lastEnd = 0;
+      let perFile = 0;
+      for (const { match, index } of matches) {
+        if (perFile >= 2) break;
+        const start = Math.max(1, match.lineNumber - 3);
+        const end = Math.min(document.lineStarts.length, start + 6);
+        if (start <= lastEnd) continue;
+        const lineRows: string[] = [];
+        for (let line = start; line <= end; line += 1) {
+          const value = document.slice(document.lineRange(line)).replace(/\n$/, "");
+          const excerpt = excerptText(value);
+          lineRows.push(
+            `${line}: ${excerpt.text}${excerpt.truncated ? " [preview line truncated]" : ""}`,
+          );
+        }
+        const row = `${path}:${match.lineNumber} {match #${index + 1}} [source preview lines ${start}-${end}]\n${lineRows.join("\n")}`;
+        if (bytes + Buffer.byteLength(row) + 2 > maxBytes) continue;
+        rows.push(row);
+        indices.push(index + 1);
+        bytes += Buffer.byteLength(row) + 2;
+        windows += 1;
+        perFile += 1;
+        lastEnd = end;
+      }
+    } catch (error) {
+      if (signal?.aborted || (error instanceof Error && error.name === "AbortError"))
+        throw abortError();
+      if (!(error instanceof Error) || !("code" in error || error.name === "SourceDocumentError"))
+        throw error;
+      reasons.push(`${path}: preview unavailable`);
+    }
+  }
+  return { text: rows.join("\n\n"), indices, windows, filesRead, reasons };
+}

package/src/summary.ts

@@ -0,0 +1,79 @@
+import {
+  DEFAULT_RESULT_TOKEN_BUDGET,
+  ESTIMATED_CHARACTERS_PER_TOKEN,
+  MAX_RESULT_BYTES,
+  type MatchRecord,
+  type SearchSnapshot,
+} from "./types.js";
+
+const METADATA_CHARACTERS = 2_400;
+const METADATA_BYTES = 3_584;
+
+/** File navigation gets its budget before optional source samples. */
+export function formatSummary(
+  snapshot: SearchSnapshot,
+  fileLimit: number,
+  offset = 0,
+  resultTokenBudget = DEFAULT_RESULT_TOKEN_BUDGET,
+) {
+  if (!Number.isSafeInteger(fileLimit) || fileLimit <= 0)
+    throw new Error("Summary file limit must be a positive safe integer");
+  if (!Number.isSafeInteger(offset) || offset < 0 || offset > snapshot.fileCounts.size)
+    throw new Error("Summary offset is outside the file summary");
+  const files = [...snapshot.fileCounts.entries()].toSorted(
+    ([left, leftCount], [right, rightCount]) => rightCount - leftCount || left.localeCompare(right),
+  );
+  const maxCharacters = Math.max(
+    256,
+    resultTokenBudget * ESTIMATED_CHARACTERS_PER_TOKEN - METADATA_CHARACTERS,
+  );
+  const maxBytes = MAX_RESULT_BYTES - METADATA_BYTES;
+  const rows: string[] = [];
+  const shownPaths: string[] = [];
+  const firstMatches = new Map<string, { match: MatchRecord; index: number }>();
+  for (const [index, match] of snapshot.matches.entries())
+    if (!firstMatches.has(match.displayPath)) firstMatches.set(match.displayPath, { match, index });
+  let bytes = 0;
+  let characters = 0;
+  for (const [file, count] of files.slice(offset, offset + Math.min(30, fileLimit))) {
+    const row = `${file}  ${String(count).padStart(6)}`;
+    if (bytes + Buffer.byteLength(row) + 1 > maxBytes) {
+      if (!rows.length)
+        throw new Error("A file summary row exceeds the response byte budget; narrow the path");
+      break;
+    }
+    if (rows.length && characters + row.length + 1 > maxCharacters) break;
+    rows.push(row);
+    shownPaths.push(file);
+    bytes += Buffer.byteLength(row) + 1;
+    characters += row.length + 1;
+  }
+  const previews: string[] = [];
+  const sampleIndices: number[] = [];
+  const sampleBudget = Math.max(0, Math.min(maxBytes - bytes, maxCharacters - characters));
+  let sampleBytes = 0;
+  for (const path of shownPaths.slice(0, 5)) {
+    const retained = firstMatches.get(path);
+    if (!retained) continue;
+    const preview = `${path}:${retained.match.lineNumber} {match #${retained.index + 1}} ${retained.match.lineContent}`;
+    if (sampleBytes + Buffer.byteLength(preview) + 1 > sampleBudget) continue;
+    previews.push(preview);
+    sampleIndices.push(retained.index + 1);
+    sampleBytes += Buffer.byteLength(preview) + 1;
+  }
+  const nextOffset = offset + rows.length;
+  return {
+    body: rows.join("\n"),
+    previews: previews.join("\n"),
+    previewsShown: previews.length,
+    previewsOmitted: shownPaths.length - previews.length,
+    shown: rows.length,
+    offset,
+    nextOffset,
+    hasNext: nextOffset < files.length,
+    omitted: files.length - nextOffset,
+    shownPaths,
+    sampleIndices,
+    previewByteBudget: sampleBudget,
+  };
+}