sift-light 1.0.0

package/src/git-process.ts

@@ -0,0 +1,189 @@
+import { SiftLightError } from "./errors.js";
+import { runOwnedProcess } from "./owned-process.js";
+import { MAX_PROTOCOL_LINE_BYTES } from "./types.js";
+
+const GIT_READ_ARGUMENTS = [
+  "--no-pager",
+  "--no-replace-objects",
+  "--no-optional-locks",
+  "-c",
+  "core.fsmonitor=false",
+  "-c",
+  "core.untrackedCache=false",
+  "-c",
+  "submodule.recurse=false",
+] as const;
+
+const MINIMUM_NO_LAZY_FETCH_VERSION = [2, 45, 0] as const;
+const gitCapabilities = new Map<string, boolean>();
+
+type GitReadCommand = "rev-parse" | "ls-tree" | "cat-file" | "ls-files" | "check-ignore";
+
+interface GitReadOptions {
+  signal?: AbortSignal;
+  input?: Uint8Array;
+  maxBytes?: number;
+  allowedCodes?: readonly number[];
+  executable?: string;
+}
+
+/** Retain ordinary ignore configuration, but never inherit Git repository or execution overrides. */
+export function gitReadEnvironment(): NodeJS.ProcessEnv {
+  return {
+    ...Object.fromEntries(
+      Object.entries(process.env).filter(([key]) => !key.toUpperCase().startsWith("GIT_")),
+    ),
+    GIT_CONFIG_COUNT: "0",
+    GIT_NO_LAZY_FETCH: "1",
+    GIT_NO_REPLACE_OBJECTS: "1",
+    GIT_OPTIONAL_LOCKS: "0",
+    GIT_TERMINAL_PROMPT: "0",
+    GIT_PROTOCOL_FROM_USER: "0",
+    LC_ALL: "C",
+  };
+}
+
+export function supportsNoLazyFetch(version: string): boolean {
+  const match = /^git version (\d+)\.(\d+)(?:\.(\d+))?/.exec(version.trim());
+  if (!match) throw new SiftLightError("Git returned an unrecognized version string");
+  const actual = [Number(match[1]), Number(match[2]), Number(match[3] ?? 0)];
+  for (let index = 0; index < MINIMUM_NO_LAZY_FETCH_VERSION.length; index += 1) {
+    const difference = (actual[index] ?? 0) - (MINIMUM_NO_LAZY_FETCH_VERSION[index] ?? 0);
+    if (difference !== 0) return difference > 0;
+  }
+  return true;
+}
+
+async function gitReadArguments(
+  executable: string,
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<string[]> {
+  const capabilityKey = `${executable}\0${process.env.PATH ?? ""}`;
+  let supports = gitCapabilities.get(capabilityKey);
+  if (supports === undefined) {
+    const versionChunks: Buffer[] = [];
+    const version = await runOwnedProcess(
+      {
+        executable,
+        args: ["--version"],
+        cwd,
+        env: gitReadEnvironment(),
+        ...(signal ? { signal } : {}),
+      },
+      async (stdout) => {
+        for await (const chunk of stdout) versionChunks.push(Buffer.from(chunk));
+      },
+    );
+    if (version.code !== 0) throw new SiftLightError("Unable to determine the Git version");
+    supports = supportsNoLazyFetch(Buffer.concat(versionChunks).toString("utf8"));
+    gitCapabilities.set(capabilityKey, supports);
+  }
+  if (supports) {
+    return [...GIT_READ_ARGUMENTS, "--no-lazy-fetch"];
+  }
+
+  const partial = await runOwnedProcess(
+    {
+      executable,
+      args: [
+        ...GIT_READ_ARGUMENTS,
+        "config",
+        "--local",
+        "--get-regexp",
+        "^(extensions\\.partialClone|remote\\..*\\.promisor)$",
+      ],
+      cwd,
+      env: gitReadEnvironment(),
+      ...(signal ? { signal } : {}),
+    },
+    async (stdout) => {
+      for await (const chunk of stdout) {
+        // Presence is represented by the exit code; content is intentionally discarded.
+        void chunk;
+      }
+    },
+  );
+  if (partial.code === 0) {
+    throw new SiftLightError(
+      "Git 2.45 or newer is required for non-fetching reads from a partial/promisor clone",
+    );
+  }
+  if (partial.code !== 1) {
+    throw new SiftLightError("Unable to verify whether this older Git repository is partial");
+  }
+  return [...GIT_READ_ARGUMENTS];
+}
+
+/** Only raw-object and names-only builtins are allowed at this boundary. */
+export async function runGitRead(
+  cwd: string,
+  command: GitReadCommand,
+  args: string[],
+  options: GitReadOptions = {},
+): Promise<{ output: Buffer; code: number }> {
+  const chunks: Buffer[] = [];
+  if (options.input && options.input.byteLength > MAX_PROTOCOL_LINE_BYTES) {
+    throw new SiftLightError(
+      `Git input exceeds the ${String(MAX_PROTOCOL_LINE_BYTES)} byte protocol limit`,
+    );
+  }
+  let bytes = 0;
+  const maxBytes = options.maxBytes ?? MAX_PROTOCOL_LINE_BYTES;
+  const result = await runOwnedProcess(
+    {
+      executable: options.executable ?? "git",
+      args: [
+        ...(await gitReadArguments(options.executable ?? "git", cwd, options.signal)),
+        ...(command === "ls-tree" ? ["--literal-pathspecs"] : []),
+        command,
+        ...args,
+      ],
+      cwd,
+      env: gitReadEnvironment(),
+      ...(options.signal ? { signal: options.signal } : {}),
+      ...(options.input ? { input: options.input } : {}),
+    },
+    async (stdout) => {
+      for await (const chunk of stdout) {
+        bytes += chunk.byteLength;
+        if (bytes > maxBytes) {
+          throw new SiftLightError(
+            `Git ${command} output exceeds the ${String(maxBytes)} byte limit`,
+          );
+        }
+        chunks.push(Buffer.from(chunk));
+      }
+    },
+  );
+  if (result.code === null || !(options.allowedCodes ?? [0]).includes(result.code)) {
+    throw new SiftLightError(
+      `Git ${command} failed: ${result.stderr.trim() || `exit ${String(result.code)}`}`,
+    );
+  }
+  return { output: Buffer.concat(chunks), code: result.code };
+}
+
+export function decodeGitPath(bytes: Buffer): string {
+  const value = bytes.toString("utf8");
+  if (!Buffer.from(value, "utf8").equals(bytes)) {
+    throw new SiftLightError(
+      "Git path is not valid UTF-8; path-based source access is unavailable",
+    );
+  }
+  return value;
+}
+
+export function splitGitRecords(output: Buffer): Buffer[] {
+  if (output.length === 0) return [];
+  if (output[output.length - 1] !== 0) {
+    throw new SiftLightError("Git names protocol ended without a NUL delimiter");
+  }
+  const records: Buffer[] = [];
+  let offset = 0;
+  for (let delimiter = output.indexOf(0); delimiter !== -1; delimiter = output.indexOf(0, offset)) {
+    records.push(output.subarray(offset, delimiter));
+    offset = delimiter + 1;
+  }
+  return records;
+}

package/src/git-repository.ts

@@ -0,0 +1,367 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, open } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { abortError, SiftLightError } from "./errors.js";
+import { decodeGitPath, runGitRead, splitGitRecords } from "./git-process.js";
+import {
+  assertExistingPathInsideCwd,
+  isPathInsideCwd,
+  sameSourceRevision,
+  sourceRevisionFromStats,
+} from "./source.js";
+import { MAX_SOURCE_FILE_BYTES, MAX_SOURCE_REVISION_FILES, type SourceRevision } from "./types.js";
+
+export interface GitTreeEntry {
+  path: string;
+  mode: string;
+  blob: string;
+  size: number;
+}
+
+export type GitSourceOrigin =
+  | { kind: "git"; commit: string; blob: string }
+  | { kind: "worktree"; revision: SourceRevision; contentHash: string };
+
+export interface GitRawSource {
+  path: string;
+  mode: string;
+  sourceStatus: "available" | "absent" | "binary" | "symlink" | "submodule" | "unavailable";
+  content?: Buffer;
+  contentHash?: string;
+  origin?: GitSourceOrigin;
+  reason?: string;
+}
+
+export interface GitReadBudget {
+  bytes: number;
+  maxBytes: number;
+}
+
+export async function verifyWorktreeRevision(
+  cwd: string,
+  path: string,
+  expected: SourceRevision,
+): Promise<void> {
+  try {
+    const current = await lstat(resolve(cwd, path));
+    await assertExistingPathInsideCwd(resolve(cwd, path), cwd);
+    if (current.isFile() && sameSourceRevision(sourceRevisionFromStats(current), expected)) return;
+  } catch (error) {
+    if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
+  }
+  throw new SiftLightError("Working source changed during Git comparison; retry a new search");
+}
+
+export function gitPath(cwd: string, path: string): string {
+  if (path.length === 0 || path.includes("\0"))
+    throw new SiftLightError("Git source path is invalid");
+  const absolute = resolve(cwd, path);
+  const local = relative(resolve(cwd), absolute).split(sep).join("/");
+  if (
+    !isPathInsideCwd(absolute, cwd) ||
+    local.split("/").some((part) => part.toLowerCase() === ".git")
+  ) {
+    throw new SiftLightError(
+      "Git source path must stay within the working directory and outside .git",
+    );
+  }
+  return local;
+}
+
+export async function resolveGitCommit(
+  cwd: string,
+  ref: string,
+  signal?: AbortSignal,
+): Promise<string> {
+  if (ref.trim().length === 0 || ref.length > 1024 || ref.includes("\0")) {
+    throw new SiftLightError("Git commit reference must be a nonempty bounded string");
+  }
+  const { output } = await runGitRead(
+    cwd,
+    "rev-parse",
+    ["--verify", "--end-of-options", `${ref}^{commit}`],
+    {
+      ...(signal ? { signal } : {}),
+      maxBytes: 128,
+    },
+  );
+  const commit = output.toString("ascii").trim();
+  if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(commit))
+    throw new SiftLightError("Git returned an invalid commit identity");
+  return commit;
+}
+
+export async function resolveGitRepository(cwd: string, signal?: AbortSignal): Promise<string> {
+  const { output } = await runGitRead(
+    cwd,
+    "rev-parse",
+    ["--show-toplevel"],
+    signal ? { signal, maxBytes: 4096 } : { maxBytes: 4096 },
+  );
+  const root = decodeGitPath(output).replace(/\r?\n$/, "");
+  if (!isAbsolute(root)) throw new SiftLightError("Git returned an invalid repository root");
+  return resolve(root);
+}
+
+export async function findGitRepository(
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<string | undefined> {
+  try {
+    return await resolveGitRepository(cwd, signal);
+  } catch (error) {
+    if (error instanceof SiftLightError && error.message.includes("not a git repository")) {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+export async function readGitTree(
+  cwd: string,
+  commit: string,
+  signal?: AbortSignal,
+  path?: string,
+): Promise<{ entries: Map<string, GitTreeEntry>; limited: boolean }> {
+  const { output } = await runGitRead(
+    cwd,
+    "ls-tree",
+    ["-r", "-z", "-l", commit, ...(path ? ["--", gitPath(cwd, path)] : [])],
+    signal ? { signal } : {},
+  );
+  const entries = new Map<string, GitTreeEntry>();
+  for (const record of splitGitRecords(output)) {
+    if (entries.size === MAX_SOURCE_REVISION_FILES) return { entries, limited: true };
+    const tab = record.indexOf(9);
+    const header = record.subarray(0, tab).toString("ascii").trim().split(/\s+/);
+    const [mode, type, blob, size] = header;
+    if (
+      tab < 0 ||
+      !mode ||
+      !blob ||
+      !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(blob) ||
+      !["blob", "commit"].includes(type ?? "")
+    ) {
+      throw new SiftLightError("Git tree returned an invalid raw object entry");
+    }
+    const local = gitPath(cwd, decodeGitPath(record.subarray(tab + 1)));
+    const byteSize = type === "commit" ? 0 : Number(size);
+    if (!Number.isSafeInteger(byteSize) || byteSize < 0)
+      throw new SiftLightError("Git tree returned an invalid blob size");
+    entries.set(local, { path: local, mode, blob, size: byteSize });
+  }
+  return { entries, limited: false };
+}
+
+export async function worktreeNames(
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<{ paths: string[]; limited: boolean }> {
+  const { output } = await runGitRead(
+    cwd,
+    "ls-files",
+    ["-z", "--cached", "--others", "--exclude-standard"],
+    signal ? { signal } : {},
+  );
+  const paths = new Set<string>();
+  for (const record of splitGitRecords(output)) {
+    if (paths.size === MAX_SOURCE_REVISION_FILES) return { paths: [...paths], limited: true };
+    paths.add(gitPath(cwd, decodeGitPath(record)));
+  }
+  return { paths: [...paths], limited: false };
+}
+
+/** --no-index deliberately applies today's privacy rules to historically tracked files as well. */
+/* oxlint-disable no-await-in-loop -- batches bound input and process concurrency; predicates may have their own shared budget. */
+export async function visibleGitPaths(
+  cwd: string,
+  paths: string[],
+  signal?: AbortSignal,
+  includePath?: (path: string) => boolean | Promise<boolean>,
+): Promise<string[]> {
+  const result: string[] = [];
+  for (let start = 0; start < paths.length; start += 128) {
+    const batch = paths.slice(start, start + 128);
+    const { output } = await runGitRead(cwd, "check-ignore", ["--no-index", "-z", "--stdin"], {
+      input: Buffer.from(`${batch.map((path) => `./${path}`).join("\0")}\0`),
+      allowedCodes: [0, 1],
+      ...(signal ? { signal } : {}),
+    });
+    const ignored = new Set(
+      splitGitRecords(output).map((record) => gitPath(cwd, decodeGitPath(record))),
+    );
+    for (const path of batch) {
+      if (signal?.aborted) throw abortError();
+      if (!ignored.has(path) && (!includePath || (await includePath(path)))) result.push(path);
+    }
+  }
+  return result;
+}
+
+/* oxlint-enable no-await-in-loop */
+
+function limitedSource(path: string, mode: string, reason: string): GitRawSource {
+  return { path, mode, sourceStatus: "unavailable", reason };
+}
+
+export async function readGitBlob(
+  cwd: string,
+  commit: string,
+  entry: GitTreeEntry,
+  budget: GitReadBudget,
+  signal?: AbortSignal,
+): Promise<GitRawSource> {
+  const { path, mode, blob, size } = entry;
+  if (mode === "120000" || mode === "160000") {
+    return {
+      path,
+      mode,
+      sourceStatus: mode === "120000" ? "symlink" : "submodule",
+      reason: "Symlink and submodule contents are not followed",
+    };
+  }
+  if (size > MAX_SOURCE_FILE_BYTES)
+    return limitedSource(
+      path,
+      mode,
+      `Source exceeds the ${String(MAX_SOURCE_FILE_BYTES)} byte file limit`,
+    );
+  if (budget.bytes + size > budget.maxBytes)
+    return limitedSource(
+      path,
+      mode,
+      `Source reads exceed the ${String(budget.maxBytes)} byte request limit`,
+    );
+  const { output } = await runGitRead(cwd, "cat-file", ["blob", blob], {
+    maxBytes: size,
+    ...(signal ? { signal } : {}),
+  });
+  budget.bytes += output.length;
+  if (output.length !== size)
+    throw new SiftLightError("Git blob size does not match its immutable tree entry");
+  const verifiedBlob = createHash(blob.length === 40 ? "sha1" : "sha256")
+    .update(`blob ${String(output.length)}\0`)
+    .update(output)
+    .digest("hex");
+  if (verifiedBlob !== blob)
+    throw new SiftLightError("Git blob bytes do not match their immutable object identity");
+  return {
+    path,
+    mode,
+    sourceStatus: output.includes(0) ? "binary" : "available",
+    ...(output.includes(0) ? { reason: "Binary source contains NUL bytes" } : { content: output }),
+    origin: { kind: "git", commit, blob },
+    contentHash: createHash("sha256").update(output).digest("hex"),
+  };
+}
+
+/* oxlint-disable no-await-in-loop -- bounded positional reads consume one file handle in order. */
+export async function readWorktreeSource(
+  cwd: string,
+  path: string,
+  budget: GitReadBudget,
+  signal?: AbortSignal,
+): Promise<GitRawSource> {
+  const absolute = resolve(cwd, path);
+  if (signal?.aborted) throw abortError();
+  let discovered = false;
+  try {
+    const before = await lstat(absolute);
+    discovered = true;
+    if (before.isSymbolicLink())
+      return {
+        path,
+        mode: "120000",
+        sourceStatus: "symlink",
+        reason: "Symlink source is not followed",
+      };
+    if (!before.isFile())
+      return {
+        path,
+        mode: "160000",
+        sourceStatus: before.isDirectory() ? "submodule" : "unavailable",
+        reason: "Non-regular source is not read",
+      };
+    const mode = (before.mode & 0o111) === 0 ? "100644" : "100755";
+    if (before.size > MAX_SOURCE_FILE_BYTES)
+      return limitedSource(
+        path,
+        mode,
+        `Source exceeds the ${String(MAX_SOURCE_FILE_BYTES)} byte file limit`,
+      );
+    if (budget.bytes + before.size > budget.maxBytes)
+      return limitedSource(
+        path,
+        mode,
+        `Source reads exceed the ${String(budget.maxBytes)} byte request limit`,
+      );
+    await assertExistingPathInsideCwd(absolute, cwd);
+    const handle = await open(absolute, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    try {
+      if (
+        !sameSourceRevision(
+          sourceRevisionFromStats(before),
+          sourceRevisionFromStats(await handle.stat()),
+        )
+      )
+        throw new SiftLightError("Working source changed before reading");
+      const buffer = Buffer.alloc(before.size + 1);
+      let bytes = 0;
+      while (bytes < buffer.length) {
+        if (signal?.aborted) throw abortError();
+        const { bytesRead } = await handle.read(
+          buffer,
+          bytes,
+          Math.min(64 * 1024, buffer.length - bytes),
+          null,
+        );
+        if (bytesRead === 0) break;
+        bytes += bytesRead;
+      }
+      budget.bytes += bytes;
+      const after = await lstat(absolute);
+      await assertExistingPathInsideCwd(absolute, cwd);
+      if (
+        bytes !== before.size ||
+        !sameSourceRevision(sourceRevisionFromStats(before), sourceRevisionFromStats(after)) ||
+        !sameSourceRevision(
+          sourceRevisionFromStats(before),
+          sourceRevisionFromStats(await handle.stat()),
+        )
+      ) {
+        throw new SiftLightError(
+          "Working source changed while reading; Git ranges and source cannot be mixed",
+        );
+      }
+      const content = buffer.subarray(0, bytes);
+      return {
+        path,
+        mode,
+        sourceStatus: content.includes(0) ? "binary" : "available",
+        ...(content.includes(0) ? { reason: "Binary source contains NUL bytes" } : { content }),
+        origin: {
+          kind: "worktree",
+          revision: sourceRevisionFromStats(after),
+          contentHash: createHash("sha256").update(content).digest("hex"),
+        },
+        contentHash: createHash("sha256").update(content).digest("hex"),
+      };
+    } finally {
+      await handle.close();
+    }
+  } catch (error) {
+    if (error instanceof Error && "code" in error) {
+      if (error.code === "ENOENT") {
+        if (discovered)
+          throw new SiftLightError("Working source disappeared while reading; retry a new search");
+        return { path, mode: "000000", sourceStatus: "absent" };
+      }
+      if (["EACCES", "EPERM", "ELOOP", "ENOTDIR"].includes(String(error.code)))
+        return limitedSource(path, "000000", `Source unavailable: ${String(error.code)}`);
+    }
+    throw error;
+  }
+}
+
+/* oxlint-enable no-await-in-loop */