trajectories 0.0.0 → 0.1.2

package/legacy/src/diagnostics.ts

@@ -0,0 +1,87 @@
+import type { IngestDiagnosticAggregate } from "./types.js";
+
+type IngestDiagnostic = { code: string; message: string; count?: number };
+
+// Upstream emits one diagnostic per record, so a stored sample keeps only the
+// last message per code, bounded so no single message can inflate a row.
+const DIAGNOSTIC_SAMPLE_CODE_POINTS = 200;
+
+/**
+ * Bound a diagnostic message to the stored sample length. Counting code points,
+ * not UTF-16 units, keeps an astral character at the boundary whole instead of
+ * leaving a lone surrogate behind. Empty messages store as null.
+ */
+export function capDiagnosticSample(message: string): string | null {
+  if (!message) return null;
+  const codePoints = Array.from(message);
+  return codePoints.length > DIAGNOSTIC_SAMPLE_CODE_POINTS
+    ? codePoints.slice(0, DIAGNOSTIC_SAMPLE_CODE_POINTS).join("")
+    : message;
+}
+
+// `count` is upstream's own aggregation when it reports one (e.g. records
+// affected by timestamps_interpolated). Zero is a real observation and must
+// survive; only an absent or nonsensical count falls back to one occurrence.
+function occurrenceCountOf(diagnostic: IngestDiagnostic): number {
+  const count = diagnostic.count;
+  return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 ? count : 1;
+}
+
+/**
+ * Collapse normalization and conversion diagnostics into one entry per code.
+ * Diagnostic text never carries raw transcript content (Trajectory's canonical
+ * contract), so the last message per code is safe to store as a bounded sample.
+ */
+export function aggregateIngestDiagnostics(
+  diagnostics: readonly IngestDiagnostic[],
+  observedAt = new Date().toISOString(),
+): IngestDiagnosticAggregate[] {
+  const byCode = new Map<string, IngestDiagnosticAggregate>();
+  for (const diagnostic of diagnostics) {
+    const count = occurrenceCountOf(diagnostic);
+    const sample = capDiagnosticSample(diagnostic.message);
+    const existing = byCode.get(diagnostic.code);
+    if (existing) {
+      existing.occurrenceCount += count;
+      existing.sample = sample;
+    } else {
+      byCode.set(diagnostic.code, {
+        code: diagnostic.code,
+        occurrenceCount: count,
+        sample,
+        firstSeenAt: observedAt,
+      });
+    }
+  }
+  return [...byCode.values()];
+}
+
+/**
+ * Union two aggregates by code, as a staged replacement accumulates the chunks
+ * that produced it: counts add (a chunk was observed once), the later sample
+ * wins, and the earliest observation is kept. `undefined` means "no
+ * information" and is never invented from an empty group.
+ */
+export function mergeIngestDiagnosticAggregates(
+  carried: readonly IngestDiagnosticAggregate[] | undefined,
+  incoming: readonly IngestDiagnosticAggregate[] | undefined,
+): IngestDiagnosticAggregate[] | undefined {
+  if (!carried && !incoming) return undefined;
+  const merged = new Map<string, IngestDiagnosticAggregate>();
+  for (const entry of carried ?? []) {
+    merged.set(entry.code, { ...entry });
+  }
+  for (const entry of incoming ?? []) {
+    const existing = merged.get(entry.code);
+    if (!existing) {
+      merged.set(entry.code, { ...entry });
+      continue;
+    }
+    existing.occurrenceCount += entry.occurrenceCount;
+    if (entry.sample) existing.sample = entry.sample;
+    if (entry.firstSeenAt && (!existing.firstSeenAt || entry.firstSeenAt < existing.firstSeenAt)) {
+      existing.firstSeenAt = entry.firstSeenAt;
+    }
+  }
+  return [...merged.values()];
+}

package/legacy/src/discovery.ts

@@ -0,0 +1,56 @@
+import type { DiscoveredSession } from "./types.js";
+
+/** Resumable discovery, distinct from each session's normalization cursor. */
+export interface DiscoveryCheckpoint {
+  source: string;
+  root_key: string;
+  cursor: string | null;
+  status: "pending" | "exhausted" | "unknown";
+  degraded: boolean;
+  last_completed_at: string | null;
+  updated_at: string;
+}
+export interface DiscoverySummary {
+  total_sources: number;
+  pending_sources: number;
+  unknown_sources: number;
+  exhausted_sources: number;
+  /** Oldest completed sweep, null until every recorded source has completed one. */
+  oldest_completed_at: string | null;
+}
+
+function identity(item: DiscoveredSession): string {
+  return JSON.stringify([item.path, item.id]);
+}
+export function uniqueDiscovered(items: DiscoveredSession[]): DiscoveredSession[] {
+  const seen = new Set<string>();
+  return items.filter(item => {
+    const key = identity(item);
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+export function discoveryHeadBudget(limit: number, cursor: string | null): number {
+  return cursor && limit > 1 ? Math.min(32, Math.max(1, Math.floor(limit / 4))) : 0;
+}
+
+/**
+ * Native scanners already produce a metadata inventory. Page it by stable
+ * identity, not mutable mtime/offset; reserve a small head lane for freshness.
+ * Metadata enumeration itself remains isolated by the preparation deadline.
+ */
+export function pageDiscoveryInventory(
+  inventory: DiscoveredSession[], limit: number, cursor: string | null,
+): { items: DiscoveredSession[]; next: string | null } {
+  if (!Number.isSafeInteger(limit) || limit < 1) throw new RangeError("Discovery limit must be positive.");
+  const after = cursor?.startsWith("k:") ? cursor.slice(2) : null;
+  const head = inventory.slice(0, discoveryHeadBudget(limit, cursor));
+  const ordered = inventory.map(item => ({ item, key: identity(item) }))
+    .filter(entry => after === null || entry.key > after)
+    .sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0);
+  const historical = ordered.slice(0, limit - head.length);
+  const next = historical.length && historical.length < ordered.length
+    ? `k:${historical.at(-1)!.key}` : null;
+  return { items: uniqueDiscovered([...head, ...historical.map(entry => entry.item)]), next };
+}

package/legacy/src/embeddings.ts

@@ -0,0 +1,382 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import type * as Ort from "onnxruntime-node";
+let ort: typeof Ort;
+
+export const DEFAULT_MODEL_NAME = "all-MiniLM-L6-v2";
+export const EMBEDDING_DIMENSION = 384;
+
+const HF_MODEL_URL =
+  "https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model_quantized.onnx";
+const HF_VOCAB_URL =
+  "https://huggingface.co/Xenova/all-MiniLM-L6-v2/raw/main/vocab.txt";
+
+export interface EmbeddingProvider {
+  name: string;
+  dimension: number;
+  embed(texts: string[]): Promise<Float32Array[]>;
+}
+
+export function getModelsDir(): string {
+  return (
+    process.env.LEDGER_MODELS_DIR ||
+    join(homedir(), ".cache", "ledger", "models")
+  );
+}
+
+export function getModelPath(): string {
+  const baseDir = getModelsDir();
+  // Check both direct path and subfolder
+  const nestedPath = join(baseDir, DEFAULT_MODEL_NAME, "model_quantized.onnx");
+  if (existsSync(nestedPath)) return nestedPath;
+
+  const directPath = join(baseDir, "model_quantized.onnx");
+  if (existsSync(directPath)) return directPath;
+
+  return nestedPath;
+}
+
+export function getVocabPath(): string {
+  const baseDir = getModelsDir();
+  const nestedPath = join(baseDir, DEFAULT_MODEL_NAME, "vocab.txt");
+  if (existsSync(nestedPath)) return nestedPath;
+
+  const directPath = join(baseDir, "vocab.txt");
+  if (existsSync(directPath)) return directPath;
+
+  // Check bundled vocab in package
+  const bundledPath = join(import.meta.dir, "vocab.txt");
+  if (existsSync(bundledPath)) return bundledPath;
+
+  return nestedPath;
+}
+
+export async function pullModel(options: { verbose?: boolean } = {}): Promise<{
+  modelPath: string;
+  vocabPath: string;
+}> {
+  const modelPath = getModelPath();
+  const vocabPath = getVocabPath();
+  const modelDir = dirname(modelPath);
+
+  if (!existsSync(modelDir)) {
+    mkdirSync(modelDir, { recursive: true });
+  }
+
+  if (!existsSync(modelPath)) {
+    if (options.verbose) {
+      console.log(`Downloading embedding model from Hugging Face to ${modelPath}...`);
+    }
+    const res = await fetch(HF_MODEL_URL, { redirect: "follow" });
+    if (!res.ok) {
+      throw new Error(
+        `Failed to download model from ${HF_MODEL_URL}: ${res.status} ${res.statusText}`,
+      );
+    }
+    const buffer = await res.arrayBuffer();
+    writeFileSync(modelPath, Buffer.from(buffer));
+    if (options.verbose) {
+      console.log(`Saved model (${(buffer.byteLength / (1024 * 1024)).toFixed(1)} MB) to ${modelPath}`);
+    }
+  }
+
+  if (!existsSync(vocabPath)) {
+    // Check bundled first before downloading
+    const bundledPath = join(import.meta.dir, "vocab.txt");
+    if (existsSync(bundledPath)) {
+      const vocabContent = readFileSync(bundledPath);
+      writeFileSync(vocabPath, vocabContent);
+    } else {
+      if (options.verbose) {
+        console.log(`Downloading vocabulary from ${HF_VOCAB_URL}...`);
+      }
+      const res = await fetch(HF_VOCAB_URL, { redirect: "follow" });
+      if (!res.ok) {
+        throw new Error(
+          `Failed to download vocab from ${HF_VOCAB_URL}: ${res.status} ${res.statusText}`,
+        );
+      }
+      const text = await res.text();
+      writeFileSync(vocabPath, text, "utf-8");
+    }
+  }
+
+  return { modelPath, vocabPath };
+}
+
+export async function ensureModelFiles(options: { verbose?: boolean } = {}): Promise<{
+  modelPath: string;
+  vocabPath: string;
+}> {
+  const modelPath = getModelPath();
+  const vocabPath = getVocabPath();
+
+  if (existsSync(modelPath) && existsSync(vocabPath)) {
+    return { modelPath, vocabPath };
+  }
+
+  return pullModel(options);
+}
+
+class WordPieceTokenizer {
+  private vocab = new Map<string, number>();
+  private unkId = 100;
+  private clsId = 101;
+  private sepId = 102;
+  private padId = 0;
+
+  constructor(vocabPath: string) {
+    const vocabText = readFileSync(vocabPath, "utf-8");
+    const lines = vocabText.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trim();
+      if (line) this.vocab.set(line, i);
+    }
+    this.unkId = this.vocab.get("[UNK]") ?? 100;
+    this.clsId = this.vocab.get("[CLS]") ?? 101;
+    this.sepId = this.vocab.get("[SEP]") ?? 102;
+    this.padId = this.vocab.get("[PAD]") ?? 0;
+  }
+
+  tokenize(text: string): number[] {
+    const cleaned = text
+      .toLowerCase()
+      .normalize("NFD")
+      .replace(/[\u0300-\u036f]/g, "");
+    const words = cleaned.match(/[\p{L}\p{N}]+|[^\s\p{L}\p{N}]/gu) ?? [];
+    const tokens: number[] = [this.clsId];
+
+    for (const word of words) {
+      let start = 0;
+      const subTokens: number[] = [];
+      while (start < word.length) {
+        let end = word.length;
+        let curId: number | null = null;
+        while (start < end) {
+          let sub = word.slice(start, end);
+          if (start > 0) sub = "##" + sub;
+          if (this.vocab.has(sub)) {
+            curId = this.vocab.get(sub)!;
+            break;
+          }
+          end--;
+        }
+        if (curId === null) {
+          subTokens.push(this.unkId);
+          break;
+        }
+        subTokens.push(curId);
+        start = end;
+      }
+      tokens.push(...subTokens);
+    }
+    tokens.push(this.sepId);
+    return tokens;
+  }
+
+  get PAD_ID(): number {
+    return this.padId;
+  }
+}
+
+export class OnnxEmbeddingProvider implements EmbeddingProvider {
+  readonly name = DEFAULT_MODEL_NAME;
+  readonly dimension = EMBEDDING_DIMENSION;
+
+  private session: Ort.InferenceSession | null = null;
+  private tokenizer: WordPieceTokenizer | null = null;
+  private initPromise: Promise<void> | null = null;
+
+  private async init(): Promise<void> {
+    if (this.session && this.tokenizer) return;
+    if (this.initPromise) return this.initPromise;
+
+    this.initPromise = (async () => {
+      const { modelPath, vocabPath } = await ensureModelFiles();
+      this.tokenizer = new WordPieceTokenizer(vocabPath);
+      ort ??= await import("onnxruntime-node");
+      this.session = await ort.InferenceSession.create(modelPath);
+    })();
+
+    return this.initPromise;
+  }
+
+  async embed(texts: string[]): Promise<Float32Array[]> {
+    if (texts.length === 0) return [];
+    await this.init();
+
+    // Chunk to bound ONNX memory: whole-session batches OOM on large
+    // sessions (observed: 4GB+ RSS, cgroup kill). 32 x 256 tokens is a safe
+    // ceiling on a 4 vCPU/15GB host.
+    const CHUNK = 32;
+    const out: Float32Array[] = [];
+    for (let start = 0; start < texts.length; start += CHUNK) {
+      const chunk = await this.embedBatch(texts.slice(start, start + CHUNK));
+      out.push(...chunk);
+    }
+    return out;
+  }
+
+  private async embedBatch(texts: string[]): Promise<Float32Array[]> {
+    if (texts.length === 0) return [];
+    await this.init();
+
+    const tokenizer = this.tokenizer!;
+    const session = this.session!;
+
+    const maxSeqLen = 256;
+    const tokenized = texts.map((t) => tokenizer.tokenize(t).slice(0, maxSeqLen));
+    const maxLen = Math.max(1, ...tokenized.map((t) => t.length));
+    const batchSize = texts.length;
+
+    const inputIds = new BigInt64Array(batchSize * maxLen);
+    const attentionMask = new BigInt64Array(batchSize * maxLen);
+    const tokenTypeIds = new BigInt64Array(batchSize * maxLen);
+
+    for (let i = 0; i < batchSize; i++) {
+      const tokens = tokenized[i];
+      for (let j = 0; j < maxLen; j++) {
+        const idx = i * maxLen + j;
+        if (j < tokens.length) {
+          inputIds[idx] = BigInt(tokens[j]);
+          attentionMask[idx] = 1n;
+        } else {
+          inputIds[idx] = BigInt(tokenizer.PAD_ID);
+          attentionMask[idx] = 0n;
+        }
+        tokenTypeIds[idx] = 0n;
+      }
+    }
+
+    const feeds = {
+      input_ids: new ort.Tensor("int64", inputIds, [batchSize, maxLen]),
+      attention_mask: new ort.Tensor("int64", attentionMask, [batchSize, maxLen]),
+      token_type_ids: new ort.Tensor("int64", tokenTypeIds, [batchSize, maxLen]),
+    };
+
+    const results = await session.run(feeds);
+    const hiddenState = results.last_hidden_state.data as Float32Array;
+    const hiddenDim = this.dimension;
+
+    const embeddings: Float32Array[] = [];
+    for (let i = 0; i < batchSize; i++) {
+      const pooled = new Float32Array(hiddenDim);
+      let tokenCount = 0;
+      for (let j = 0; j < maxLen; j++) {
+        if (attentionMask[i * maxLen + j] === 1n) {
+          tokenCount++;
+          const offset = i * maxLen * hiddenDim + j * hiddenDim;
+          for (let d = 0; d < hiddenDim; d++) {
+            pooled[d] += hiddenState[offset + d];
+          }
+        }
+      }
+      const count = Math.max(tokenCount, 1);
+      let normSq = 0;
+      for (let d = 0; d < hiddenDim; d++) {
+        pooled[d] /= count;
+        normSq += pooled[d] * pooled[d];
+      }
+      const norm = Math.sqrt(normSq);
+      if (norm > 0) {
+        for (let d = 0; d < hiddenDim; d++) {
+          pooled[d] /= norm;
+        }
+      }
+      embeddings.push(pooled);
+    }
+
+    return embeddings;
+  }
+}
+
+let activeProvider: EmbeddingProvider = new OnnxEmbeddingProvider();
+
+export function getEmbeddingProvider(): EmbeddingProvider {
+  return activeProvider;
+}
+
+export function setEmbeddingProvider(provider: EmbeddingProvider): void {
+  activeProvider = provider;
+}
+
+export async function embed(texts: string[]): Promise<Float32Array[]> {
+  return activeProvider.embed(texts);
+}
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length) return 0;
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+  }
+  return dot;
+}
+
+export function serializeEmbedding(vec: Float32Array): Uint8Array {
+  return new Uint8Array(vec.buffer, vec.byteOffset, vec.byteLength);
+}
+
+export function deserializeEmbedding(bytes: Uint8Array | Buffer): Float32Array {
+  return new Float32Array(
+    bytes.buffer,
+    bytes.byteOffset,
+    bytes.byteLength / 4,
+  );
+}
+
+/**
+ * Harness-injected boilerplate wrappers: big machine-generated preamble
+ * records (plugin catalogs, environment contexts) that match every query
+ * and carry no human/agent intent. Never embed these.
+ */
+const BOILERPLATE_PREFIXES = [
+  "<recommended_plugins>",
+  "<environment_context>",
+  "<user_info>",
+  "<system_warning>",
+];
+
+export function isBoilerplateRecord(content: string | null | undefined): boolean {
+  if (!content) return false;
+  const t = content.trimStart();
+  return BOILERPLATE_PREFIXES.some((p) => t.startsWith(p));
+}
+
+export function getRecordEmbeddingText(r: {
+  record_type: string;
+  content?: string | null;
+  tool_name?: string | null;
+  tool_arguments_json?: string | null;
+  tool_result_json?: string | null;
+}): string | null {
+  const parts: string[] = [];
+
+  if (r.content && r.content.trim() && !isBoilerplateRecord(r.content)) {
+    parts.push(r.content.trim());
+  }
+
+  if (r.tool_name) {
+    let toolStr = `Tool: ${r.tool_name}`;
+    if (r.tool_arguments_json) {
+      const truncatedArgs =
+        r.tool_arguments_json.length > 500
+          ? r.tool_arguments_json.slice(0, 500) + "..."
+          : r.tool_arguments_json;
+      toolStr += ` (${truncatedArgs})`;
+    }
+    parts.push(toolStr);
+  }
+
+  if (r.record_type === "tool" && r.tool_result_json && !r.content) {
+    const truncatedRes =
+      r.tool_result_json.length > 500
+        ? r.tool_result_json.slice(0, 500) + "..."
+        : r.tool_result_json;
+    parts.push(`Result: ${truncatedRes}`);
+  }
+
+  const text = parts.join("\n").trim();
+  return text.length > 0 ? text : null;
+}