sift-light 1.0.0

package/src/source-continuations.ts

@@ -0,0 +1,168 @@
+import { randomUUID } from "node:crypto";
+import {
+  ANALYSIS_TTL_MS,
+  MAX_SOURCE_CONTINUATIONS,
+  MAX_SOURCE_CONTINUATION_BYTES,
+} from "./analysis-limits.js";
+import { CursorError } from "./errors.js";
+import type { ByteRange, SourceReference } from "./source-document.js";
+import { mergeByteRanges, subtractByteRange } from "./source-pages.js";
+import type { SourceBoundary } from "./types.js";
+
+interface Continuation {
+  id: string;
+  source: SourceReference;
+  target: ByteRange[];
+  gaps: ByteRange[];
+  boundary?: SourceBoundary;
+  accessed: number;
+  /** Only cursors issued by this chain are valid; offsets are not a public seek API. */
+  issued: Set<number>;
+}
+
+export interface ResolvedSourceContinuation {
+  source: SourceReference;
+  target: ByteRange[];
+  remaining: ByteRange[];
+  boundary?: SourceBoundary;
+}
+
+/** Cursor offsets encode progress, not mutable per-page state; retries can replay. */
+export class SourceContinuations {
+  readonly #items = new Map<string, Continuation>();
+  readonly #now: () => number;
+
+  constructor(now: () => number = Date.now) {
+    this.#now = now;
+  }
+
+  create(
+    source: SourceReference,
+    target: readonly ByteRange[],
+    gaps: readonly ByteRange[],
+    boundary?: SourceBoundary,
+  ): string {
+    this.#sweep();
+    const item: Continuation = {
+      id: randomUUID(),
+      source: structuredClone(source),
+      target: mergeByteRanges(target),
+      gaps: mergeByteRanges(gaps),
+      ...(boundary ? { boundary } : {}),
+      accessed: this.#now(),
+      issued: new Set([0]),
+    };
+    if (
+      item.gaps.length === 0 ||
+      item.gaps.some(
+        (gap) =>
+          gap.start === gap.end ||
+          !item.target.some((range) => range.start <= gap.start && gap.end <= range.end),
+      )
+    ) {
+      throw new CursorError("Source continuation requires missing ranges inside its target");
+    }
+    this.#items.set(item.id, item);
+    while (
+      this.#items.size > MAX_SOURCE_CONTINUATIONS ||
+      Buffer.byteLength(
+        JSON.stringify(
+          [...this.#items.values()].map((continuation) =>
+            Object.assign(
+              {
+                id: continuation.id,
+                source: continuation.source,
+                target: continuation.target,
+                gaps: continuation.gaps,
+                accessed: continuation.accessed,
+                issued: [...continuation.issued],
+              },
+              continuation.boundary ? { boundary: continuation.boundary } : {},
+            ),
+          ),
+        ),
+      ) > MAX_SOURCE_CONTINUATION_BYTES
+    ) {
+      let oldest: Continuation | undefined;
+      for (const candidate of this.#items.values()) {
+        if (!oldest || candidate.accessed < oldest.accessed) oldest = candidate;
+      }
+      if (!oldest) break;
+      this.#items.delete(oldest.id);
+    }
+    if (!this.#items.has(item.id))
+      throw new CursorError("Source continuation metadata exceeds its limit");
+    return `${item.id}.source.0`;
+  }
+
+  resolve(cursor: string): ResolvedSourceContinuation {
+    const { item, consumed } = this.#resolve(cursor);
+    return {
+      source: structuredClone(item.source),
+      target: item.target.map((range) => ({ ...range })),
+      remaining: this.#remaining(item, consumed),
+      ...(item.boundary ? { boundary: item.boundary } : {}),
+    };
+  }
+
+  advance(cursor: string, returned: ByteRange): string | undefined {
+    const { item, consumed } = this.#resolve(cursor);
+    const remaining = this.#remaining(item, consumed);
+    const first = remaining[0];
+    if (
+      !first ||
+      returned.start !== first.start ||
+      returned.end <= returned.start ||
+      returned.end > first.end
+    ) {
+      throw new CursorError("Source continuation must advance along its next missing range");
+    }
+    const next = consumed + returned.end - returned.start;
+    item.issued.add(next);
+    return this.#remaining(item, next).length > 0
+      ? `${item.id}.source.${next.toString(36)}`
+      : undefined;
+  }
+
+  clear(): void {
+    this.#items.clear();
+  }
+
+  #resolve(cursor: string): { item: Continuation; consumed: number } {
+    this.#sweep();
+    const match = /^([0-9a-f-]+)\.source\.([0-9a-z]+)$/.exec(cursor);
+    if (!match?.[1] || !match[2]) throw new CursorError("Invalid source continuation cursor");
+    const item = this.#items.get(match[1]);
+    if (!item) throw new CursorError("Source continuation expired or was evicted; inspect again");
+    const consumed = Number.parseInt(match[2], 36);
+    const length = item.gaps.reduce((sum, range) => sum + range.end - range.start, 0);
+    if (
+      !Number.isSafeInteger(consumed) ||
+      consumed < 0 ||
+      consumed >= length ||
+      !item.issued.has(consumed)
+    ) {
+      throw new CursorError("Source continuation offset is outside its missing ranges");
+    }
+    item.accessed = this.#now();
+    return { item, consumed };
+  }
+
+  #remaining(item: Continuation, consumed: number): ByteRange[] {
+    let left = consumed;
+    let ranges = item.gaps.map((range) => ({ ...range }));
+    for (const range of item.gaps) {
+      if (left === 0) break;
+      const take = Math.min(left, range.end - range.start);
+      ranges = subtractByteRange(ranges, { start: range.start, end: range.start + take });
+      left -= take;
+    }
+    return ranges;
+  }
+
+  #sweep(): void {
+    const cutoff = this.#now() - ANALYSIS_TTL_MS;
+    for (const item of this.#items.values())
+      if (item.accessed < cutoff) this.#items.delete(item.id);
+  }
+}

package/src/source-document.ts

@@ -0,0 +1,277 @@
+import { isUtf8 } from "node:buffer";
+import { createHash } from "node:crypto";
+import { open, realpath } from "node:fs/promises";
+import { relative, resolve } from "node:path";
+import { abortError, SiftLightError } from "./errors.js";
+import { isPathInsideCwd, SearchPathPolicy } from "./path-policy.js";
+import { getSourceRevision, sameSourceRevision, sourceRevisionFromStats } from "./source.js";
+import { MAX_SOURCE_FILE_BYTES, type SourceRevision } from "./types.js";
+
+export type SourceOrigin =
+  | { kind: "worktree"; revision: SourceRevision; contentHash: string }
+  | { kind: "git"; commit: string; blob: string };
+
+export interface SourceReference {
+  path: string;
+  origin: SourceOrigin;
+}
+
+export interface ByteRange {
+  start: number;
+  end: number;
+}
+
+export interface SourcePosition {
+  line: number;
+  /** One-based UTF-16 column, with the byte range remaining authoritative. */
+  column: number;
+}
+
+export class SourceDocumentError extends SiftLightError {
+  readonly reason: "source-changed" | "source-unavailable" | "file-too-large" | "encoding";
+
+  constructor(
+    reason: "source-changed" | "source-unavailable" | "file-too-large" | "encoding",
+    message: string,
+  ) {
+    super(message);
+    this.reason = reason;
+    this.name = "SourceDocumentError";
+  }
+}
+
+export function contentHash(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+/** One verified read. Byte offsets refer to the original, unnormalized source. */
+export class SourceDocument {
+  readonly reference: SourceReference;
+  readonly bytes: Buffer;
+  readonly text: string;
+  readonly utf8: boolean;
+  readonly lineStarts: number[] = [0];
+  #byteOffsets?: Uint32Array;
+
+  constructor(reference: SourceReference, bytes: Buffer) {
+    if (
+      typeof reference !== "object" ||
+      reference === null ||
+      typeof reference.path !== "string" ||
+      reference.path.length === 0 ||
+      typeof reference.origin !== "object" ||
+      reference.origin === null ||
+      !("kind" in reference.origin) ||
+      (reference.origin.kind !== "git" && reference.origin.kind !== "worktree")
+    ) {
+      throw new SourceDocumentError("source-unavailable", "Source reference is invalid");
+    }
+    if (!Buffer.isBuffer(bytes)) {
+      throw new SourceDocumentError("source-unavailable", "Source bytes are invalid");
+    }
+    this.reference = reference;
+    this.bytes = bytes;
+    if (bytes.length > MAX_SOURCE_FILE_BYTES) {
+      throw new SourceDocumentError("file-too-large", "Source exceeds the 5 MiB file limit");
+    }
+    this.utf8 = isUtf8(bytes);
+    this.text = bytes.toString("utf8");
+    for (let index = bytes.indexOf(10); index >= 0; index = bytes.indexOf(10, index + 1)) {
+      this.lineStarts.push(index + 1);
+    }
+  }
+
+  get path(): string {
+    return this.reference.path;
+  }
+
+  toByteOffset(character: number): number {
+    this.#requireUtf8();
+    if (!Number.isSafeInteger(character) || character < 0 || character > this.text.length) {
+      throw new SiftLightError("Source character offset is outside the document");
+    }
+    const code = this.text.charCodeAt(character);
+    if (code >= 0xdc00 && code <= 0xdfff) {
+      throw new SiftLightError("Source character offset splits a Unicode character");
+    }
+    const value = this.#offsets()[character];
+    if (value === undefined) throw new Error("Missing source offset");
+    return value;
+  }
+
+  toCharacterOffset(byte: number): number {
+    this.#requireUtf8();
+    this.checkRange({ start: byte, end: byte });
+    const offsets = this.#offsets();
+    let low = 0;
+    let high = offsets.length - 1;
+    while (low < high) {
+      const middle = Math.floor((low + high) / 2);
+      const value = offsets[middle];
+      if (value === undefined) throw new Error("Missing source offset");
+      if (value < byte) low = middle + 1;
+      else high = middle;
+    }
+    if (offsets[low] !== byte) {
+      throw new SiftLightError("Source byte offset splits a Unicode character");
+    }
+    return low;
+  }
+
+  lineAt(byte: number): number {
+    this.checkRange({ start: byte, end: byte });
+    let low = 0;
+    let high = this.lineStarts.length;
+    while (low + 1 < high) {
+      const middle = Math.floor((low + high) / 2);
+      const start = this.lineStarts[middle];
+      if (start === undefined) throw new Error("Missing source line");
+      if (start <= byte) low = middle;
+      else high = middle;
+    }
+    return low + 1;
+  }
+
+  positionAt(byte: number): SourcePosition {
+    const line = this.lineAt(byte);
+    const start = this.lineStarts[line - 1];
+    if (start === undefined) throw new Error("Missing source line");
+    return {
+      line,
+      column: this.toCharacterOffset(byte) - this.toCharacterOffset(start) + 1,
+    };
+  }
+
+  lineRange(startLine: number, endLine = startLine): ByteRange {
+    if (
+      !Number.isSafeInteger(startLine) ||
+      !Number.isSafeInteger(endLine) ||
+      startLine < 1 ||
+      endLine < startLine ||
+      startLine > this.lineStarts.length
+    ) {
+      throw new SiftLightError("Source line range is outside the document");
+    }
+    const start = this.lineStarts[startLine - 1];
+    if (start === undefined) throw new Error("Missing source line");
+    return { start, end: this.lineStarts[endLine] ?? this.bytes.length };
+  }
+
+  slice(range: ByteRange): string {
+    this.#requireUtf8();
+    this.checkRange(range);
+    this.toCharacterOffset(range.start);
+    this.toCharacterOffset(range.end);
+    return this.bytes.subarray(range.start, range.end).toString("utf8");
+  }
+
+  checkRange(range: ByteRange): void {
+    if (
+      !Number.isSafeInteger(range.start) ||
+      !Number.isSafeInteger(range.end) ||
+      range.start < 0 ||
+      range.end < range.start ||
+      range.end > this.bytes.length
+    ) {
+      throw new SiftLightError("Source byte range is outside the document");
+    }
+  }
+
+  #requireUtf8(): void {
+    if (!this.utf8) {
+      throw new SourceDocumentError("encoding", "Source is not losslessly representable as UTF-8");
+    }
+  }
+
+  #offsets(): Uint32Array {
+    if (this.#byteOffsets) return this.#byteOffsets;
+    const offsets = new Uint32Array(this.text.length + 1);
+    let character = 0;
+    let byte = 0;
+    for (const point of this.text) {
+      offsets[character] = byte;
+      if (point.length === 2) offsets[character + 1] = byte;
+      character += point.length;
+      byte += Buffer.byteLength(point);
+    }
+    offsets[character] = byte;
+    this.#byteOffsets = offsets;
+    return offsets;
+  }
+}
+
+export async function readWorkspaceDocument(
+  path: string,
+  cwd: string,
+  signal?: AbortSignal,
+  expected?: SourceOrigin,
+  readBudget = MAX_SOURCE_FILE_BYTES,
+): Promise<SourceDocument> {
+  if (signal?.aborted) throw abortError();
+  if (expected?.kind === "git") {
+    throw new SiftLightError("A Git source reference cannot be read from the worktree");
+  }
+  const absolute = resolve(cwd, path);
+  const [canonical, canonicalCwd] = await Promise.all([
+    new SearchPathPolicy(cwd).resolveExistingPath(absolute),
+    realpath(cwd),
+  ]);
+  if (!canonical) throw new SourceDocumentError("source-unavailable", "Source is unavailable");
+  const before = await getSourceRevision(absolute);
+  if (!before) throw new SourceDocumentError("source-unavailable", "Source is unavailable");
+  if (expected && !sameSourceRevision(before, expected.revision)) {
+    throw new SourceDocumentError("source-changed", "Source changed; start a new inspection");
+  }
+  if (before.size > Math.min(MAX_SOURCE_FILE_BYTES, readBudget)) {
+    throw new SourceDocumentError("file-too-large", "Source exceeds the 5 MiB file limit");
+  }
+  if (signal?.aborted) throw abortError();
+  const handle = await open(canonical, "r");
+  let bytes: Buffer;
+  try {
+    const metadata = await handle.stat();
+    if (!metadata.isFile()) {
+      throw new SourceDocumentError("source-unavailable", "Source must be a regular file");
+    }
+    if (!sameSourceRevision(before, sourceRevisionFromStats(metadata))) {
+      throw new SourceDocumentError("source-changed", "Source was replaced before reading");
+    }
+    const buffer = Buffer.alloc(before.size);
+    let used = 0;
+    while (used < buffer.length) {
+      if (signal?.aborted) throw abortError();
+      // oxlint-disable-next-line no-await-in-loop -- each positional read advances the same file buffer.
+      const read = await handle.read(buffer, used, buffer.length - used, used);
+      if (read.bytesRead === 0) break;
+      used += read.bytesRead;
+    }
+    if (used !== before.size) {
+      throw new SourceDocumentError("source-changed", "Source changed during reading");
+    }
+    bytes = buffer.subarray(0, used);
+  } finally {
+    await handle.close();
+  }
+  const [after, finalPath] = await Promise.all([getSourceRevision(absolute), realpath(absolute)]);
+  if (!after || canonical !== finalPath || !sameSourceRevision(before, after)) {
+    throw new SourceDocumentError("source-changed", "Source changed during reading");
+  }
+  if (signal?.aborted) throw abortError();
+  const hash = contentHash(bytes);
+  if (expected && expected.contentHash !== hash) {
+    throw new SourceDocumentError(
+      "source-changed",
+      "Source content changed; start a new inspection",
+    );
+  }
+  return new SourceDocument(
+    {
+      // Source references are protocol paths, so keep them POSIX across host platforms.
+      path: isPathInsideCwd(canonical, canonicalCwd)
+        ? relative(canonicalCwd, canonical).replaceAll("\\", "/")
+        : canonical.replaceAll("\\", "/"),
+      origin: { kind: "worktree", revision: after, contentHash: hash },
+    },
+    bytes,
+  );
+}