sift-light 1.0.0

package/src/concept-embedding-cache.ts

@@ -0,0 +1,186 @@
+import { createHash, randomUUID } from "node:crypto";
+import type { Dirent } from "node:fs";
+import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  CONCEPT_CACHE_MAX_BYTES,
+  CONCEPT_CACHE_VERSION,
+  CONCEPT_EMBEDDING_DIMENSIONS,
+} from "./concept-model.js";
+import { isRecordValue } from "./record-value.js";
+
+export interface CachedConceptWindow {
+  start: number;
+  end: number;
+  vector: number[];
+}
+
+export interface CachedConceptEmbedding {
+  key: string;
+  windows: CachedConceptWindow[];
+}
+
+interface StoredEmbedding {
+  version: number;
+  key: string;
+  windows: { start: number; end: number; vector: string }[];
+}
+
+function errorCode(error: unknown): unknown {
+  return typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
+}
+
+export function conceptEmbeddingKey(role: "query" | "passage", text: string): string {
+  return createHash("sha256").update(role).update("\0").update(text).digest("hex");
+}
+
+function cachePath(root: string, key: string): string {
+  return join(root, key.slice(0, 2), `${key}.json`);
+}
+
+function encodeVector(vector: readonly number[]): string {
+  const buffer = Buffer.allocUnsafe(vector.length * Float32Array.BYTES_PER_ELEMENT);
+  for (const [index, value] of vector.entries())
+    buffer.writeFloatLE(value, index * Float32Array.BYTES_PER_ELEMENT);
+  return buffer.toString("base64");
+}
+
+function decodeVector(encoded: string): number[] | undefined {
+  const buffer = Buffer.from(encoded, "base64");
+  if (buffer.length !== CONCEPT_EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT)
+    return undefined;
+  const vector = Array.from({ length: CONCEPT_EMBEDDING_DIMENSIONS }, (_, index) =>
+    buffer.readFloatLE(index * Float32Array.BYTES_PER_ELEMENT),
+  );
+  return vector.every(Number.isFinite) ? vector : undefined;
+}
+
+function storedEmbedding(value: unknown, key: string): CachedConceptEmbedding | undefined {
+  if (
+    !isRecordValue(value) ||
+    value.version !== CONCEPT_CACHE_VERSION ||
+    value.key !== key ||
+    !Array.isArray(value.windows) ||
+    value.windows.length === 0
+  )
+    return undefined;
+  const windows: CachedConceptWindow[] = [];
+  for (const item of value.windows) {
+    if (
+      !isRecordValue(item) ||
+      typeof item.start !== "number" ||
+      !Number.isSafeInteger(item.start) ||
+      item.start < 0 ||
+      typeof item.end !== "number" ||
+      !Number.isSafeInteger(item.end) ||
+      item.end <= item.start ||
+      typeof item.vector !== "string"
+    )
+      return undefined;
+    const vector = decodeVector(item.vector);
+    if (!vector) return undefined;
+    windows.push({ start: item.start, end: item.end, vector });
+  }
+  return { key, windows };
+}
+
+export async function readConceptEmbedding(
+  root: string,
+  key: string,
+): Promise<CachedConceptEmbedding | undefined> {
+  try {
+    return storedEmbedding(JSON.parse(await readFile(cachePath(root, key), "utf8")), key);
+  } catch (error) {
+    const code = errorCode(error);
+    if (code === "ENOENT" || error instanceof SyntaxError) return undefined;
+    throw error;
+  }
+}
+
+export async function writeConceptEmbedding(
+  root: string,
+  embedding: CachedConceptEmbedding,
+  stagingRoot = root,
+): Promise<void> {
+  const directory = join(root, embedding.key.slice(0, 2));
+  await mkdir(stagingRoot, { recursive: true });
+  await mkdir(directory, { recursive: true });
+  const destination = cachePath(root, embedding.key);
+  const temporary = join(stagingRoot, `.${embedding.key}.${randomUUID()}.tmp`);
+  const stored: StoredEmbedding = {
+    version: CONCEPT_CACHE_VERSION,
+    key: embedding.key,
+    windows: embedding.windows.map((window) => ({
+      start: window.start,
+      end: window.end,
+      vector: encodeVector(window.vector),
+    })),
+  };
+  try {
+    await writeFile(temporary, JSON.stringify(stored), { flag: "wx", mode: 0o600 });
+    try {
+      await rename(temporary, destination);
+    } catch (error) {
+      if (errorCode(error) !== "EEXIST") throw error;
+      if (await readConceptEmbedding(root, embedding.key)) return;
+      await rm(destination, { force: true });
+      try {
+        await rename(temporary, destination);
+      } catch (replacementError) {
+        if (errorCode(replacementError) !== "EEXIST") throw replacementError;
+      }
+    }
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+export async function enforceConceptCacheLimit(
+  root: string,
+  maximumBytes = CONCEPT_CACHE_MAX_BYTES,
+): Promise<number> {
+  const entries: { path: string; bytes: number; modified: number }[] = [];
+  let shards: Dirent[];
+  try {
+    shards = await readdir(root, { withFileTypes: true });
+  } catch (error) {
+    if (errorCode(error) === "ENOENT") return 0;
+    throw error;
+  }
+  for (const shard of shards) {
+    if (!shard.isDirectory() || !/^[0-9a-f]{2}$/u.test(shard.name)) continue;
+    const directory = join(root, shard.name);
+    let files: Dirent[];
+    try {
+      // oxlint-disable-next-line no-await-in-loop -- bounded 256-shard cache inventory.
+      files = await readdir(directory, { withFileTypes: true });
+    } catch (error) {
+      if (errorCode(error) === "ENOENT") continue;
+      throw error;
+    }
+    for (const file of files) {
+      if (!file.isFile() || !file.name.endsWith(".json")) continue;
+      const path = join(directory, file.name);
+      let metadata;
+      try {
+        // oxlint-disable-next-line no-await-in-loop -- eviction needs authoritative file sizes and mtimes.
+        metadata = await stat(path);
+      } catch (error) {
+        if (errorCode(error) === "ENOENT") continue;
+        throw error;
+      }
+      entries.push({ path, bytes: metadata.size, modified: metadata.mtimeMs });
+    }
+  }
+  let bytes = entries.reduce((total, item) => total + item.bytes, 0);
+  if (bytes <= maximumBytes) return bytes;
+  for (const item of entries.toSorted(
+    (a, b) => a.modified - b.modified || a.path.localeCompare(b.path),
+  )) {
+    // oxlint-disable-next-line no-await-in-loop -- deterministic oldest-first eviction owns its files.
+    await rm(item.path, { force: true });
+    bytes -= item.bytes;
+    if (bytes <= maximumBytes) break;
+  }
+  return bytes;
+}

package/src/concept-inference.ts

@@ -0,0 +1,218 @@
+import type { FeatureExtractionPipeline } from "@huggingface/transformers";
+import type { CachedConceptEmbedding } from "./concept-embedding-cache.js";
+import {
+  CONCEPT_EMBEDDING_DIMENSIONS,
+  CONCEPT_MODEL_TOKENS,
+  CONCEPT_WINDOW_OVERLAP_TOKENS,
+} from "./concept-model.js";
+
+export interface PendingConceptEmbedding {
+  key: string;
+  role: "query" | "passage";
+  text: string;
+}
+
+export interface ConceptEmbeddingCallbacks {
+  onBatch?: (completed: number, total: number) => void;
+  onCompletedEmbedding?: (
+    embedding: CachedConceptEmbedding,
+    completed: number,
+    total: number,
+  ) => void | Promise<void>;
+}
+
+const INFERENCE_BATCH_SIZE = 16;
+
+/** Snap a candidate UTF-16 index so it never splits a surrogate pair. */
+export function safeUtf16End(text: string, end: number): number {
+  if (end <= 0 || end >= text.length) return end;
+  const previous = text.charCodeAt(end - 1);
+  return previous >= 0xd800 && previous <= 0xdbff ? end - 1 : end;
+}
+
+function tokenCount(extractor: FeatureExtractionPipeline, text: string): number {
+  return extractor.tokenizer.encode(text).length;
+}
+
+/**
+ * Binary-search `low` must strictly advance even when `safeUtf16End`
+ * retreats a midpoint below the current bound (surrogate-pair interiors).
+ */
+function advanceLow(low: number, middle: number): number {
+  return Math.max(low + 1, middle + 1);
+}
+
+function nextUtf16Boundary(text: string, start: number): number {
+  const next = start + 1;
+  if (next >= text.length) return next;
+  const first = text.charCodeAt(start);
+  const second = text.charCodeAt(next);
+  return first >= 0xd800 && first <= 0xdbff && second >= 0xdc00 && second <= 0xdfff
+    ? next + 1
+    : next;
+}
+
+function binarySearchBudget(span: number): number {
+  return 2 * Math.max(span, 1) + 32;
+}
+
+export function maximumTokenSafeEnd(
+  extractor: FeatureExtractionPipeline,
+  prefix: string,
+  text: string,
+  start: number,
+): number {
+  let low = start + 1;
+  let high = text.length;
+  let accepted = start;
+  let iterations = 0;
+  const maxIterations = binarySearchBudget(high - low + 1);
+  while (low <= high) {
+    iterations += 1;
+    if (iterations > maxIterations)
+      throw new Error("Concept token window search failed to make progress");
+    const middle = safeUtf16End(text, Math.floor((low + high) / 2));
+    if (middle <= start) {
+      low += 1;
+      continue;
+    }
+    if (tokenCount(extractor, `${prefix}${text.slice(start, middle)}`) <= CONCEPT_MODEL_TOKENS) {
+      accepted = middle;
+      low = advanceLow(low, middle);
+    } else {
+      high = middle - 1;
+    }
+  }
+  return accepted;
+}
+
+export function overlapStart(
+  extractor: FeatureExtractionPipeline,
+  prefix: string,
+  text: string,
+  start: number,
+  end: number,
+): number {
+  let low = start + 1;
+  let high = end;
+  let accepted = end;
+  let iterations = 0;
+  const maxIterations = binarySearchBudget(high - low + 1);
+  while (low <= high) {
+    iterations += 1;
+    if (iterations > maxIterations)
+      throw new Error("Concept overlap search failed to make progress");
+    const middle = safeUtf16End(text, Math.floor((low + high) / 2));
+    const tokens = tokenCount(extractor, `${prefix}${text.slice(middle, end)}`);
+    if (tokens <= CONCEPT_WINDOW_OVERLAP_TOKENS) {
+      accepted = middle;
+      high = middle - 1;
+    } else {
+      low = advanceLow(low, middle);
+    }
+  }
+  return Math.max(nextUtf16Boundary(text, start), accepted);
+}
+
+export function tokenSafeWindows(
+  extractor: FeatureExtractionPipeline,
+  pending: PendingConceptEmbedding,
+): { start: number; end: number; input: string }[] {
+  const prefix = `${pending.role}: `;
+  const windows: { start: number; end: number; input: string }[] = [];
+  let start = 0;
+  while (start < pending.text.length) {
+    const end = maximumTokenSafeEnd(extractor, prefix, pending.text, start);
+    if (end <= start)
+      throw new Error("Concept tokenizer could not admit one complete source character");
+    windows.push({ start, end, input: `${prefix}${pending.text.slice(start, end)}` });
+    if (end >= pending.text.length) break;
+    const next = overlapStart(extractor, prefix, pending.text, start, end);
+    if (next <= start)
+      throw new Error("Concept window overlap failed to advance past a complete window");
+    start = next;
+  }
+  return windows;
+}
+
+export async function embedConceptInputs(
+  extractor: FeatureExtractionPipeline,
+  pending: readonly PendingConceptEmbedding[],
+  callbacks: ConceptEmbeddingCallbacks = {},
+): Promise<CachedConceptEmbedding[]> {
+  const layouts = pending.map((item) => tokenSafeWindows(extractor, item));
+  const inputs = layouts.flatMap((windows) => windows.map((window) => window.input));
+  const vectors: number[][] = [];
+  const layoutEnds: number[] = [];
+  let layoutEnd = 0;
+  for (const windows of layouts) {
+    layoutEnd += windows.length;
+    layoutEnds.push(layoutEnd);
+  }
+  let completedEmbedding = 0;
+  for (let offset = 0; offset < inputs.length; offset += INFERENCE_BATCH_SIZE) {
+    const batch = inputs.slice(offset, offset + INFERENCE_BATCH_SIZE);
+    // oxlint-disable-next-line no-await-in-loop -- bounded batches cap native tensor memory.
+    const tensor = await extractor(batch, { pooling: "mean", normalize: true });
+    if (tensor.data.length !== batch.length * CONCEPT_EMBEDDING_DIMENSIONS)
+      throw new Error("Unexpected concept embedding dimensions");
+    for (let index = 0; index < batch.length; index += 1) {
+      const start = index * CONCEPT_EMBEDDING_DIMENSIONS;
+      vectors.push(
+        Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number),
+      );
+    }
+    const completedWindows = Math.min(offset + batch.length, inputs.length);
+    while (
+      layoutEnds[completedEmbedding] !== undefined &&
+      layoutEnds[completedEmbedding]! <= completedWindows
+    ) {
+      const start = completedEmbedding === 0 ? 0 : layoutEnds[completedEmbedding - 1]!;
+      const item = pending[completedEmbedding];
+      if (!item) throw new Error("Missing completed concept embedding input");
+      // oxlint-disable-next-line no-await-in-loop -- cache writes must complete at each embedding boundary.
+      await callbacks.onCompletedEmbedding?.(
+        {
+          key: item.key,
+          windows: (layouts[completedEmbedding] ?? []).map((window, index) => ({
+            start: window.start,
+            end: window.end,
+            vector: vectors[start + index] ?? [],
+          })),
+        },
+        completedEmbedding + 1,
+        pending.length,
+      );
+      completedEmbedding += 1;
+    }
+    callbacks.onBatch?.(completedWindows, inputs.length);
+  }
+  let vectorIndex = 0;
+  return pending.map((item, index) => ({
+    key: item.key,
+    windows: (layouts[index] ?? []).map((window) => ({
+      start: window.start,
+      end: window.end,
+      vector: vectors[vectorIndex++] ?? [],
+    })),
+  }));
+}
+
+export function cosineSimilarity(left: readonly number[], right: readonly number[]): number {
+  if (left.length !== CONCEPT_EMBEDDING_DIMENSIONS || right.length !== left.length)
+    throw new Error("Inconsistent concept embedding dimensions");
+  const score = left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
+  if (!Number.isFinite(score)) throw new Error("Non-finite concept similarity");
+  return score;
+}
+
+export function meanNormalized(vectors: readonly (readonly number[])[]): number[] {
+  if (!vectors.length) throw new Error("Cannot pool an empty concept embedding");
+  const pooled = Array.from({ length: CONCEPT_EMBEDDING_DIMENSIONS }, (_, index) =>
+    vectors.reduce((sum, vector) => sum + (vector[index] ?? 0), 0),
+  );
+  const magnitude = Math.sqrt(pooled.reduce((sum, value) => sum + value * value, 0));
+  if (!Number.isFinite(magnitude) || magnitude === 0)
+    throw new Error("Invalid pooled concept query embedding");
+  return pooled.map((value) => value / magnitude);
+}

package/src/concept-model.ts

@@ -0,0 +1,96 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+import { SiftLightError } from "./errors.js";
+
+export const CONCEPT_MODEL = "Xenova/multilingual-e5-small";
+export const CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78";
+export const MAX_CONCEPT_CHARS = 1_000;
+export const CONCEPT_PASSAGE_OVERLAP_CHARS = 160;
+export const CONCEPT_MODEL_TOKENS = 512;
+export const CONCEPT_WINDOW_OVERLAP_TOKENS = 64;
+export const CONCEPT_EMBEDDING_DIMENSIONS = 384;
+export const CONCEPT_CACHE_VERSION = 1;
+export const CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024;
+export const CONCEPT_TIMEOUT_MS = 10 * 60_000;
+export const MIN_CONCEPT_TIMEOUT_MS = 1_000;
+export const MAX_CONCEPT_TIMEOUT_MS = 60 * 60_000;
+export const CONCEPT_TIMEOUT_ENV = "SIFT_LIGHT_CONCEPT_TIMEOUT_MS";
+export const MAX_CONCEPT_WORKER_INPUT_BYTES = 64 * 1024 * 1024;
+export const MAX_CONCEPT_WORKER_OUTPUT_BYTES = 4 * 1024 * 1024;
+export const CONCEPT_ASSETS = [
+  {
+    path: "config.json",
+    bytes: 658,
+    sha256: "cb99455288675345e1a4f411438d5d0adbba5fbd3a67ea4fb03c015433b996c1",
+  },
+  {
+    path: "tokenizer_config.json",
+    bytes: 443,
+    sha256: "a1d6bc8734a6f635dc158508bef000f8e2e5a759c7d92f984b2c86e5ff53425b",
+  },
+  {
+    path: "tokenizer.json",
+    bytes: 17_082_730,
+    sha256: "0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39",
+  },
+  {
+    path: "onnx/model_quantized.onnx",
+    bytes: 118_308_185,
+    sha256: "f80102d3f2a1229f387d3c81909990d8945513e347b0eab049f7de3c6f98c193",
+  },
+] as const;
+export function conceptModelDirectory(): string {
+  return resolve(
+    process.env.SIFT_LIGHT_MODEL_DIR ?? join(homedir(), ".cache", "sift-light", "models"),
+    CONCEPT_REVISION,
+  );
+}
+
+export function conceptCacheDirectory(): string {
+  return resolve(
+    process.env.SIFT_LIGHT_MODEL_DIR ?? join(homedir(), ".cache", "sift-light", "models"),
+    "concept-cache",
+    `${CONCEPT_REVISION}-v${String(CONCEPT_CACHE_VERSION)}`,
+  );
+}
+
+/** Interactive hosts may lower the deadline; invalid values fail closed with an explicit error. */
+export function resolveConceptTimeoutMs(environment: NodeJS.ProcessEnv = process.env): number {
+  const raw = environment[CONCEPT_TIMEOUT_ENV];
+  if (raw === undefined || raw === "") return CONCEPT_TIMEOUT_MS;
+  const value = Number(raw);
+  if (
+    !Number.isSafeInteger(value) ||
+    value < MIN_CONCEPT_TIMEOUT_MS ||
+    value > MAX_CONCEPT_TIMEOUT_MS
+  ) {
+    throw new SiftLightError(
+      `${CONCEPT_TIMEOUT_ENV} must be an integer from ${String(MIN_CONCEPT_TIMEOUT_MS)} through ${String(MAX_CONCEPT_TIMEOUT_MS)}`,
+    );
+  }
+  return value;
+}
+export async function verifyConceptModel(directory = conceptModelDirectory()): Promise<void> {
+  for (const asset of CONCEPT_ASSETS) {
+    const hash = createHash("sha256");
+    let size = 0;
+    try {
+      // oxlint-disable-next-line no-await-in-loop -- bounded streaming integrity checks, one model asset at a time.
+      for await (const chunk of createReadStream(join(directory, asset.path))) {
+        const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        size += bytes.length;
+        if (size > asset.bytes) throw new Error("Asset exceeds pinned size");
+        hash.update(bytes);
+      }
+      if (size !== asset.bytes || hash.digest("hex") !== asset.sha256)
+        throw new Error("Pinned hash mismatch");
+    } catch (error) {
+      throw new SiftLightError(
+        `Local concept model is missing or invalid (${asset.path}); run sift-light-model --install-model explicitly`,
+        { cause: error },
+      );
+    }
+  }
+}