sentinelayer-cli 0.39.2 → 0.41.0

package/src/session/recall/embedder.js

@@ -0,0 +1,181 @@
+/**
+ * ENGRAM §1 — pluggable dense text embedder (256-d, int8, deterministic).
+ *
+ * Design fidelity (ENGRAM §4 model roster + §7 storage math):
+ *  - 256-d Matryoshka-truncated text vectors, int8-quantized for the
+ *    "exact int8 scan" recall path (§7 stage 2a). No ANN — a session is
+ *    small-data (Thesis 1), so exhaustive scan is 100% recall by definition.
+ *  - The interface is PLUGGABLE. The DEFAULT is a deterministic
+ *    hash-projection stub (the feature-hashing trick) so the 8-Needle
+ *    harness (§11) is reproducible in CI with no model download. A real
+ *    EmbeddingGemma-class embedder plugs in behind the same interface via
+ *    `provider: "api"`, degrading to the stub on any failure — the same
+ *    provider-fallback shape the repo already uses in
+ *    `src/memory/retrieval.js`.
+ *
+ * The stub is intentionally lexical-leaning: chains in Needle-Chain are
+ * linked ONLY through shared entities (their head/tail share no tokens), so
+ * chain completion is carried by the graph-expansion stage (§7 spreading
+ * activation), not by dense similarity — exactly as the design intends.
+ */
+
+import { tokenize, bigrams, charNGrams, fnv1a32 } from "./text.js";
+
+export const DEFAULT_DIM = 256;
+const QUANT_SCALE = 127;
+
+/**
+ * Deterministic hash-projection into a dense float vector, then L2-normalize.
+ * Signed feature hashing: each feature lands in one bucket with a stable
+ * sign, so collisions cancel in expectation rather than compound.
+ * @param {string} text
+ * @param {number} dim
+ * @returns {Float32Array}
+ */
+function hashProjectionFloat(text, dim) {
+  const vec = new Float32Array(dim);
+  const tokens = tokenize(text);
+  if (tokens.length === 0) return vec;
+  // Features: token unigrams + token bigrams (phrase order) + character
+  // trigrams per token (robust overlap for short texts). Char trigrams make
+  // genuine token overlap dominate collision noise in the 256-d space.
+  const features = tokens.concat(bigrams(tokens));
+  for (const token of tokens) {
+    for (const gram of charNGrams(token, 3)) features.push(`c:${gram}`);
+  }
+  for (const feature of features) {
+    const bucket = fnv1a32(feature) % dim;
+    const sign = (fnv1a32(`sign:${feature}`) & 1) === 1 ? 1 : -1;
+    vec[bucket] += sign;
+  }
+  let sumSquares = 0;
+  for (let i = 0; i < dim; i += 1) sumSquares += vec[i] * vec[i];
+  const norm = Math.sqrt(sumSquares);
+  if (norm > 0) {
+    for (let i = 0; i < dim; i += 1) vec[i] /= norm;
+  }
+  return vec;
+}
+
+/**
+ * Quantize a normalized float vector to int8 (×127, rounded, clamped).
+ * @param {Float32Array} floatVec
+ * @returns {Int8Array}
+ */
+export function quantizeInt8(floatVec) {
+  const out = new Int8Array(floatVec.length);
+  for (let i = 0; i < floatVec.length; i += 1) {
+    const scaled = Math.round(floatVec[i] * QUANT_SCALE);
+    out[i] = scaled > QUANT_SCALE ? QUANT_SCALE : scaled < -QUANT_SCALE ? -QUANT_SCALE : scaled;
+  }
+  return out;
+}
+
+/**
+ * L2 norm of an int8 vector (computed once at index time, cached for the
+ * exact-scan cosine).
+ * @param {Int8Array} vec
+ * @returns {number}
+ */
+export function normInt8(vec) {
+  let sumSquares = 0;
+  for (let i = 0; i < vec.length; i += 1) sumSquares += vec[i] * vec[i];
+  return Math.sqrt(sumSquares);
+}
+
+/**
+ * Dot product of two equal-length int8 vectors.
+ * @param {Int8Array} a
+ * @param {Int8Array} b
+ * @returns {number}
+ */
+export function dotInt8(a, b) {
+  let dot = 0;
+  const len = Math.min(a.length, b.length);
+  for (let i = 0; i < len; i += 1) dot += a[i] * b[i];
+  return dot;
+}
+
+/**
+ * Cosine similarity of two int8 vectors given precomputed norms.
+ * @returns {number} in [-1, 1]
+ */
+export function cosineInt8(a, b, normA, normB) {
+  if (!normA || !normB) return 0;
+  return dotInt8(a, b) / (normA * normB);
+}
+
+/**
+ * Create an embedder. Default is the deterministic hash-projection stub.
+ *
+ * @param {object} [options]
+ * @param {number}  [options.dim=256]
+ * @param {"stub"|"api"} [options.provider="stub"]
+ * @param {string} [options.endpoint="")]  Real-embedder HTTP endpoint (provider="api").
+ * @param {string} [options.apiKey=""]
+ * @param {Function} [options.fetchImpl]   Injected fetch implementation for tests/provider.
+ * @returns {{
+ *   name: string, dim: number,
+ *   embedFloat: (text: string) => Float32Array,
+ *   embed: (text: string) => Int8Array,
+ *   embedRemote: (text: string) => Promise<{vector: Int8Array, provider: string, fallback: boolean}>
+ * }}
+ */
+export function createEmbedder({
+  dim = DEFAULT_DIM,
+  provider = "stub",
+  endpoint = "",
+  apiKey = "",
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  const normalizedDim = Number.isInteger(dim) && dim > 0 ? dim : DEFAULT_DIM;
+  const embedFloat = (text) => hashProjectionFloat(text, normalizedDim);
+  const embed = (text) => quantizeInt8(embedFloat(text));
+
+  // Bound the remote embedder call so a hung provider can't stall recall; on
+  // timeout AbortSignal rejects the fetch and we fall back to the deterministic
+  // stub via the catch below.
+  const EMBED_REMOTE_TIMEOUT_MS = 10_000;
+  async function embedRemote(text) {
+    const canUseApi =
+      provider === "api" && endpoint && typeof fetchImpl === "function";
+    if (!canUseApi) {
+      return { vector: embed(text), provider: "stub", fallback: provider === "api" };
+    }
+    try {
+      const response = await fetchImpl(String(endpoint), {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}),
+        },
+        body: JSON.stringify({ text: String(text ?? ""), dim: normalizedDim }),
+        signal: AbortSignal.timeout(EMBED_REMOTE_TIMEOUT_MS),
+      });
+      if (!response || !response.ok) {
+        throw new Error(`embedder api ${response ? response.status : "no_response"}`);
+      }
+      const payload = await response.json();
+      const values = Array.isArray(payload?.vector) ? payload.vector : null;
+      if (!values || values.length === 0) {
+        throw new Error("embedder api returned empty vector");
+      }
+      // Matryoshka-truncate/pad to dim, then quantize for the shared scan path.
+      const floatVec = new Float32Array(normalizedDim);
+      for (let i = 0; i < normalizedDim && i < values.length; i += 1) {
+        floatVec[i] = Number(values[i]) || 0;
+      }
+      return { vector: quantizeInt8(floatVec), provider: "api", fallback: false };
+    } catch {
+      return { vector: embed(text), provider: "stub", fallback: true };
+    }
+  }
+
+  return {
+    name: provider === "api" ? `api:${normalizedDim}` : `hash-projection:${normalizedDim}`,
+    dim: normalizedDim,
+    embedFloat,
+    embed,
+    embedRemote,
+  };
+}

package/src/session/recall/index-build.js

@@ -0,0 +1,38 @@
+/**
+ * SentinelLayer session adapter for the pure ENGRAM recall-index builder.
+ *
+ * Raw session events are filtered and normalized here. Detachable ENGRAM
+ * consumers import index-core.js directly and therefore never pull the
+ * SentinelLayer session runtime into their dependency closure.
+ */
+
+import { buildObservations } from "./observations.js";
+import { buildRecallIndex as buildCoreRecallIndex } from "./index-core.js";
+
+export { BM25_K1, BM25_B } from "./index-core.js";
+
+export function buildRecallIndex({
+  events = [],
+  observations: prebuiltObservations = null,
+  messageActions = [],
+  embedder,
+  sessionId = "",
+  includeControlEvents = false,
+} = {}) {
+  const built = Array.isArray(prebuiltObservations)
+    ? {
+        observations: prebuiltObservations,
+        droppedControlEvents: 0,
+        materialCount: prebuiltObservations.length,
+      }
+    : buildObservations(events, { sessionId, includeControlEvents });
+
+  return buildCoreRecallIndex({
+    observations: built.observations,
+    messageActions,
+    embedder,
+    sessionId,
+    droppedControlEvents: built.droppedControlEvents,
+    materialCount: built.materialCount,
+  });
+}

package/src/session/recall/index-core.js

@@ -0,0 +1,108 @@
+/**
+ * Pure ENGRAM recall-index builder.
+ *
+ * This module accepts canonical observations only. Session event filtering is
+ * an adapter concern in index-build.js, which keeps the retrieval closure
+ * reusable without importing SentinelLayer session runtime.
+ */
+
+import { buildEntityGraph } from "./entities.js";
+import { createEmbedder, normInt8 } from "./embedder.js";
+
+// BM25 free parameters (Robertson/Sparck-Jones defaults — standard, not tuned).
+export const BM25_K1 = 1.2;
+export const BM25_B = 0.75;
+
+function buildBm25(observations) {
+  const postings = new Map();
+  const docLen = new Map();
+  let totalLen = 0;
+
+  for (const obs of observations) {
+    const tokens = obs.tokens;
+    docLen.set(obs.id, tokens.length);
+    totalLen += tokens.length;
+    const tf = new Map();
+    for (const token of tokens) tf.set(token, (tf.get(token) || 0) + 1);
+    for (const [term, count] of tf.entries()) {
+      let bucket = postings.get(term);
+      if (!bucket) {
+        bucket = new Map();
+        postings.set(term, bucket);
+      }
+      bucket.set(obs.id, count);
+    }
+  }
+
+  const df = new Map();
+  for (const [term, bucket] of postings.entries()) df.set(term, bucket.size);
+
+  const N = observations.length;
+  const avgdl = N > 0 ? totalLen / N : 0;
+  return { postings, df, docLen, avgdl, N };
+}
+
+function buildDense(observations, embedder) {
+  const vectors = new Map();
+  const norms = new Map();
+  for (const obs of observations) {
+    const vector = embedder.embed(obs.text);
+    vectors.set(obs.id, vector);
+    norms.set(obs.id, normInt8(vector));
+  }
+  return { vectors, norms, dim: embedder.dim };
+}
+
+/** Build a recall index from already-normalized observations. */
+export function buildRecallIndex({
+  observations = [],
+  messageActions = [],
+  embedder = createEmbedder(),
+  sessionId = "",
+  droppedControlEvents = 0,
+  materialCount = observations.length,
+} = {}) {
+  if (!Array.isArray(observations)) {
+    throw new TypeError("observations must be an array.");
+  }
+
+  const byId = new Map();
+  for (const obs of observations) byId.set(obs.id, obs);
+
+  const order = observations
+    .slice()
+    .sort((a, b) => {
+      const sa = a.sequenceId || 0;
+      const sb = b.sequenceId || 0;
+      if (sa !== sb) return sa - sb;
+      return a.index - b.index;
+    })
+    .map((obs) => obs.id);
+
+  const graph = buildEntityGraph(observations, messageActions);
+  const bm25 = buildBm25(observations);
+  const dense = buildDense(observations, embedder);
+
+  return {
+    schemaVersion: "1.0.0",
+    sessionId,
+    observations,
+    byId,
+    order,
+    graph,
+    bm25,
+    dense,
+    occurrences: graph.occurrences,
+    embedder,
+    meta: {
+      observationCount: observations.length,
+      materialCount,
+      droppedControlEvents,
+      entityCount: graph.counts.entities,
+      edgeCount: graph.counts.edges,
+      occurrenceCount: graph.counts.occurrences,
+      embedder: embedder.name,
+      dim: embedder.dim,
+    },
+  };
+}

package/src/session/recall/index.js

@@ -0,0 +1,131 @@
+/**
+ * ENGRAM §1 — Session Recall: public entry point.
+ *
+ * `runSessionRecall` is the orchestrator behind `sl session recall`:
+ *   1. (optional) full remote hydrate — tail:0 backfill so we never index a
+ *      truncated local mirror (Redis hot-caches only the latest ~2k events;
+ *      older history lives in Postgres and must be paged down first). We
+ *      honor `eventsBackfillComplete` and warn if the backfill is partial.
+ *   2. read the full local NDJSON stream (tail:0)
+ *   3. pull the message-actions layer (reply edges + ACT-R rehearsal fuel)
+ *   4. build the tri-layer recall index
+ *   5. run the Recall Engine
+ *   6. compute the token cut vs full-tail replay
+ *
+ * Runtime dependencies are injected (defaults wire the real session
+ * modules) so the orchestrator is unit-testable without network or fs.
+ */
+
+import { readStream } from "../stream.js";
+import { hydrateSessionFromRemote } from "../remote-hydrate.js";
+import { listSessionMessageActions } from "../sync.js";
+
+import { buildRecallIndex } from "./index-build.js";
+import { recall } from "./retrieve.js";
+import { createEmbedder } from "./embedder.js";
+import { computeTokenCut } from "./token-stats.js";
+
+export { buildRecallIndex } from "./index-build.js";
+export { recall, RECALL_DEFAULTS, RECALL_WEIGHTS } from "./retrieve.js";
+export { createEmbedder, DEFAULT_DIM } from "./embedder.js";
+export { buildObservations } from "./observations.js";
+export { buildEntityGraph, matchQueryEntities, EDGE_TYPES } from "./entities.js";
+export { buildProvenance } from "./provenance.js";
+export { computeTokenCut } from "./token-stats.js";
+export { runEightNeedle } from "./needle-eval.js";
+
+/**
+ * Hydrate + build + recall for a session.
+ *
+ * @param {object} params
+ * @param {string} params.sessionId
+ * @param {string} params.need           The recall query / need.
+ * @param {string} [params.targetPath]
+ * @param {number} [params.k=12]
+ * @param {string} [params.role]
+ * @param {boolean} [params.remote=true] Hydrate from the API before indexing.
+ * @param {boolean} [params.includeControlEvents=false]
+ * @param {object} [params.embedder]
+ * @param {number} [params.now]          ACT-R clock (default Date.now()).
+ * @param {Function} [params._hydrate]   seam
+ * @param {Function} [params._readStream] seam
+ * @param {Function} [params._listActions] seam
+ * @returns {Promise<object>} recall payload (JSON-serializable)
+ */
+export async function runSessionRecall({
+  sessionId,
+  need,
+  targetPath = process.cwd(),
+  k = 12,
+  role = "",
+  remote = true,
+  includeControlEvents = false,
+  embedder = createEmbedder(),
+  now = Date.now(),
+  _hydrate = hydrateSessionFromRemote,
+  _readStream = readStream,
+  _listActions = listSessionMessageActions,
+} = {}) {
+  const normalizedSessionId = String(sessionId || "").trim();
+  const normalizedNeed = String(need || "").trim();
+  if (!normalizedSessionId) {
+    return { ok: false, reason: "invalid_session_id", results: [] };
+  }
+  if (normalizedNeed.length < 2) {
+    return { ok: false, reason: "query_too_short", results: [] };
+  }
+
+  const backfill = { attempted: false, ok: false, complete: null, warning: "" };
+  if (remote) {
+    backfill.attempted = true;
+    const hydration = await _hydrate({
+      sessionId: normalizedSessionId,
+      targetPath,
+      includeControlEvents,
+    }).catch((error) => ({ ok: false, reason: String(error?.message || error) }));
+    backfill.ok = Boolean(hydration?.ok);
+    backfill.complete = hydration?.eventsBackfillComplete !== false;
+    if (!hydration?.ok) {
+      backfill.warning = `remote hydrate skipped (${hydration?.reason || "unknown"}); indexing local mirror only`;
+    } else if (hydration.eventsBackfillComplete === false) {
+      backfill.warning = `remote backfill incomplete (${hydration.eventsBackfillReason || "more pages"}); recall may miss older memories`;
+    }
+  }
+
+  const events = await _readStream(normalizedSessionId, { targetPath, tail: 0 }).catch(() => []);
+
+  let messageActions = [];
+  const actionsResult = await _listActions(normalizedSessionId, { targetPath, limit: 500 }).catch(() => null);
+  if (actionsResult?.ok && Array.isArray(actionsResult.actions)) {
+    messageActions = actionsResult.actions;
+  }
+
+  const index = buildRecallIndex({
+    events,
+    messageActions,
+    embedder,
+    sessionId: normalizedSessionId,
+    includeControlEvents,
+  });
+
+  const recalled = recall(index, { query: normalizedNeed, k, role, now });
+  const tokenCut = await computeTokenCut({
+    observations: index.observations,
+    results: recalled.results,
+  });
+
+  return {
+    ok: true,
+    reason: "",
+    sessionId: normalizedSessionId,
+    need: normalizedNeed,
+    role: role || null,
+    k: recalled.k,
+    backfill,
+    meta: index.meta,
+    matchedEntities: recalled.matchedEntityIds,
+    poolSize: recalled.poolSize,
+    results: recalled.results,
+    tokenCut,
+  };
+}