zam-core 0.6.3 → 0.7.0

package/dist/cli/index.js

@@ -656,6 +656,18 @@ CREATE TABLE IF NOT EXISTS user_config (
   updated_at  TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
+-- Token embeddings: one vector per token for semantic search (ADR 2026-07-03).
+-- Not a column on tokens: hot paths do SELECT * FROM tokens and a ~3KB blob
+-- per row would ride along on all of them.
+CREATE TABLE IF NOT EXISTS token_embeddings (
+  token_id     TEXT PRIMARY KEY REFERENCES tokens(id) ON DELETE CASCADE,
+  embedding    BLOB NOT NULL,
+  model        TEXT NOT NULL,
+  dims         INTEGER NOT NULL,
+  content_hash TEXT NOT NULL,
+  embedded_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
 -- Agent skills: task recipes the agent learns from user guidance
 CREATE TABLE IF NOT EXISTS agent_skills (
   id          TEXT PRIMARY KEY,
@@ -1002,6 +1014,16 @@ async function runMigrations(db) {
       await db.exec(`ALTER TABLE tokens ADD COLUMN topic_id TEXT`);
     }
   }
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS token_embeddings (
+      token_id     TEXT PRIMARY KEY REFERENCES tokens(id) ON DELETE CASCADE,
+      embedding    BLOB NOT NULL,
+      model        TEXT NOT NULL,
+      dims         INTEGER NOT NULL,
+      content_hash TEXT NOT NULL,
+      embedded_at  TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
 }
 
 // src/kernel/db/snapshot.ts
@@ -2410,6 +2432,151 @@ async function deleteSetting(db, key) {
   return result.changes > 0;
 }
 
+// src/kernel/models/token-embedding.ts
+import { createHash as createHash2 } from "crypto";
+function embeddingContentForToken(t2) {
+  return `${t2.concept}
+${t2.question ?? ""}
+${t2.domain}`;
+}
+function computeContentHash(text) {
+  return createHash2("sha256").update(text, "utf8").digest("hex");
+}
+function encodeEmbedding(vec) {
+  const f = Float32Array.from(vec);
+  return new Uint8Array(f.buffer);
+}
+function decodeEmbedding(blob) {
+  if (blob.byteLength % 4 !== 0) {
+    throw new Error(
+      "Invalid embedding blob size: must be a multiple of 4 bytes"
+    );
+  }
+  if (blob.byteOffset % 4 === 0) {
+    return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+  }
+  const copy = Uint8Array.prototype.slice.call(blob);
+  return new Float32Array(copy.buffer, 0, copy.byteLength / 4);
+}
+async function upsertTokenEmbedding(db, input8) {
+  const encoded = encodeEmbedding(input8.embedding);
+  const dims = input8.embedding.length;
+  const embeddedAt = (/* @__PURE__ */ new Date()).toISOString();
+  await db.prepare(`
+      INSERT INTO token_embeddings (token_id, embedding, model, dims, content_hash, embedded_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT(token_id) DO UPDATE SET
+        embedding = excluded.embedding,
+        model = excluded.model,
+        dims = excluded.dims,
+        content_hash = excluded.content_hash,
+        embedded_at = excluded.embedded_at
+    `).run(
+    input8.tokenId,
+    encoded,
+    input8.model,
+    dims,
+    input8.contentHash,
+    embeddedAt
+  );
+}
+async function listTokensNeedingEmbedding(db, model, opts) {
+  const rows = await db.prepare(`
+      SELECT t.*, e.model AS emb_model, e.dims AS emb_dims, e.content_hash AS emb_hash
+      FROM tokens t
+      LEFT JOIN token_embeddings e ON e.token_id = t.id
+      WHERE t.deprecated_at IS NULL
+    `).all();
+  const needing = [];
+  for (const row of rows) {
+    let reason = null;
+    let text = "";
+    if (opts?.force) {
+      reason = "content-changed";
+    } else if (row.emb_model === null) {
+      reason = "missing";
+    } else if (row.emb_model !== model) {
+      reason = "model-changed";
+    } else if (opts?.dims !== void 0 && row.emb_dims !== opts.dims) {
+      reason = "dimension-changed";
+    } else {
+      const computedText = embeddingContentForToken(row);
+      const hash = computeContentHash(computedText);
+      if (row.emb_hash !== hash) {
+        reason = "content-changed";
+        text = computedText;
+      }
+    }
+    if (reason) {
+      if (!text) {
+        text = embeddingContentForToken(row);
+      }
+      needing.push({ token: row, text, reason });
+    }
+  }
+  if (opts?.limit !== void 0) {
+    return needing.slice(0, opts.limit);
+  }
+  return needing;
+}
+async function getEmbeddingCoverage(db, model, opts) {
+  const rows = await db.prepare(`
+      SELECT t.*, e.model AS emb_model, e.dims AS emb_dims, e.content_hash AS emb_hash
+      FROM tokens t
+      LEFT JOIN token_embeddings e ON e.token_id = t.id
+      WHERE t.deprecated_at IS NULL
+    `).all();
+  let missing = 0;
+  let stale = 0;
+  for (const row of rows) {
+    if (row.emb_model === null) {
+      missing++;
+      continue;
+    }
+    if (row.emb_model !== model) {
+      stale++;
+      continue;
+    }
+    if (opts?.dims !== void 0 && row.emb_dims !== opts.dims) {
+      stale++;
+      continue;
+    }
+    const hash = computeContentHash(embeddingContentForToken(row));
+    if (row.emb_hash !== hash) {
+      stale++;
+    }
+  }
+  const tokens = rows.length;
+  return {
+    tokens,
+    embedded: tokens - missing - stale,
+    missing,
+    stale
+  };
+}
+async function listEmbeddedTokens(db, model) {
+  const rows = await db.prepare(`
+      SELECT t.*, e.embedding AS embedding, e.content_hash AS emb_hash
+      FROM token_embeddings e
+      JOIN tokens t ON t.id = e.token_id
+      WHERE e.model = ? AND t.deprecated_at IS NULL
+    `).all(model);
+  return rows.flatMap((row) => {
+    const { embedding, emb_hash, ...token } = row;
+    if (emb_hash !== computeContentHash(embeddingContentForToken(token))) {
+      return [];
+    }
+    try {
+      return [{ token, embedding: decodeEmbedding(embedding) }];
+    } catch (err) {
+      console.warn(
+        `Warning: Corrupted embedding for token ${token.slug} (${token.id}) ignored: ${err.message}`
+      );
+      return [];
+    }
+  });
+}
+
 // src/kernel/observation/analyzer.ts
 function parseMonitorLog(jsonl) {
   const events = [];
@@ -4546,6 +4713,82 @@ function intersperseNew(reviews, newCards, interval) {
   return result;
 }
 
+// src/kernel/search/hybrid.ts
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  const len = a.length;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+async function searchTokensHybrid(db, query, opts) {
+  const limit = opts?.limit ?? 20;
+  const rrfK = opts?.rrfK ?? 60;
+  const vectorTopK = opts?.vectorTopK ?? 10;
+  const lexicalHits = await findTokens(db, query);
+  let vectorHits = [];
+  if (opts?.queryEmbedding && opts?.model) {
+    const queryVec = Float32Array.from(opts.queryEmbedding);
+    const embedded = await listEmbeddedTokens(db, opts.model);
+    const candidates = [];
+    for (const row of embedded) {
+      if (row.embedding.length !== queryVec.length) {
+        continue;
+      }
+      const similarity = cosineSimilarity(queryVec, row.embedding);
+      if (similarity <= 0) {
+        continue;
+      }
+      candidates.push({ token: row.token, similarity });
+    }
+    candidates.sort((a, b) => b.similarity - a.similarity);
+    vectorHits = candidates.slice(0, vectorTopK);
+  }
+  const tokenMap = /* @__PURE__ */ new Map();
+  const getOrCreateEntry = (token) => {
+    let entry = tokenMap.get(token.id);
+    if (!entry) {
+      entry = {
+        ...token,
+        score: 0,
+        lexicalRank: null,
+        vectorRank: null,
+        similarity: null
+      };
+      tokenMap.set(token.id, entry);
+    }
+    return entry;
+  };
+  for (let i = 0; i < lexicalHits.length; i++) {
+    const hit = lexicalHits[i];
+    const entry = getOrCreateEntry(hit);
+    entry.lexicalRank = i + 1;
+    entry.score += 1 / (rrfK + entry.lexicalRank);
+  }
+  for (let j = 0; j < vectorHits.length; j++) {
+    const hit = vectorHits[j];
+    const entry = getOrCreateEntry(hit.token);
+    entry.vectorRank = j + 1;
+    entry.similarity = hit.similarity;
+    entry.score += 1 / (rrfK + entry.vectorRank);
+  }
+  const results = Array.from(tokenMap.values());
+  results.sort((a, b) => {
+    if (Math.abs(a.score - b.score) > 1e-9) {
+      return b.score - a.score;
+    }
+    return a.slug.localeCompare(b.slug);
+  });
+  return results.slice(0, limit);
+}
+
 // src/kernel/system/hooks.ts
 import {
   appendFileSync as appendFileSync3,
@@ -6092,6 +6335,21 @@ async function getLegacyRoleConfig(db, role, enabled) {
       maxFrames: Number.isNaN(parsed) ? 100 : parsed
     };
   }
+  if (role === "embedding") {
+    const url = await getSetting(db, "llm.embedding.url") || base.url;
+    return {
+      enabled,
+      url,
+      // Kept as a literal (not imported) to avoid a client.ts <-> embedder.ts
+      // import cycle; must match DEFAULT_EMBEDDING_MODEL in embedder.ts.
+      model: await getSetting(db, "llm.embedding.model") || "embeddinggemma",
+      apiKey: await getSetting(db, "llm.embedding.api_key") || base.apiKey,
+      apiFlavor: inferApiFlavor(url),
+      locale: base.locale,
+      source: "legacy",
+      local: isLocalEndpoint(url)
+    };
+  }
   return {
     enabled,
     url: base.url,
@@ -10110,6 +10368,219 @@ function getCurriculumProvider(id) {
   return CURRICULUM_PROVIDERS.find((provider) => provider.id === id);
 }
 
+// src/cli/llm/embedder.ts
+var CANONICAL_EMBEDDING_MODEL_ID = "embeddinggemma-300m";
+var EMBEDDINGGEMMA_ALIASES = /* @__PURE__ */ new Set([
+  "embeddinggemma",
+  "embeddinggemma:300m",
+  "embeddinggemma-300m",
+  "embed-gemma",
+  "embed-gemma:300m",
+  "google/embeddinggemma-300m"
+]);
+function canonicalEmbeddingModelId(model) {
+  const lowered = model.trim().toLowerCase();
+  if (EMBEDDINGGEMMA_ALIASES.has(lowered)) {
+    return CANONICAL_EMBEDDING_MODEL_ID;
+  }
+  return lowered;
+}
+function embeddingTextForToken(t2, model) {
+  const isGemma = EMBEDDINGGEMMA_ALIASES.has(model.trim().toLowerCase());
+  const content = embeddingContentForToken(t2);
+  return isGemma ? `title: none | text: ${content}` : content;
+}
+function embeddingTextForQuery(text, model) {
+  const isGemma = EMBEDDINGGEMMA_ALIASES.has(model.trim().toLowerCase());
+  return isGemma ? `task: search result | query: ${text}` : text;
+}
+async function resolveUsableEmbeddingEndpoint(db) {
+  const cfg = await getProviderForRole(db, "embedding");
+  if (!cfg.enabled) return null;
+  for (const endpoint of [cfg, ...cfg.fallback ? [cfg.fallback] : []]) {
+    if (endpoint.apiFlavor !== "chat-completions") continue;
+    const online = await isLlmOnline(endpoint.url);
+    if (!online) continue;
+    const availableModels = await getAvailableModels(
+      endpoint.url,
+      endpoint.apiKey
+    );
+    const modelAvailable = availableModels.length === 0 || availableModels.some(
+      (candidate) => candidate.toLowerCase() === endpoint.model.toLowerCase()
+    );
+    if (modelAvailable) return endpoint;
+  }
+  return null;
+}
+async function embedTexts(endpoint, texts) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), 6e4);
+  let res;
+  try {
+    res = await fetch(`${endpoint.url}/embeddings`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${endpoint.apiKey || DEFAULT_LLM_API_KEY}`
+      },
+      body: JSON.stringify({ model: endpoint.model, input: texts }),
+      signal: controller.signal
+    });
+  } finally {
+    clearTimeout(timeoutId);
+  }
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(
+      `Embedding request failed: ${res.statusText} (${res.status}) - ${body}`
+    );
+  }
+  const data = await res.json();
+  const items = data.data ?? [];
+  if (items.length !== texts.length) {
+    throw new Error(
+      `Embedding response returned ${items.length} vectors for ${texts.length} inputs`
+    );
+  }
+  const ordered = new Array(texts.length);
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+    const index = typeof item.index === "number" ? item.index : i;
+    if (!Number.isInteger(index)) {
+      throw new Error("Embedding response item is missing a valid index");
+    }
+    if (index < 0 || index >= texts.length) {
+      throw new Error(`Embedding response index out of range: ${index}`);
+    }
+    if (!Array.isArray(item.embedding) || item.embedding.length === 0 || !item.embedding.every((n) => typeof n === "number" && Number.isFinite(n))) {
+      throw new Error(
+        `Embedding response at index ${index} is not a non-empty array of finite numbers`
+      );
+    }
+    ordered[index] = item.embedding;
+  }
+  const firstLength = ordered[0]?.length;
+  if (ordered.some((vec) => vec === void 0 || vec.length !== firstLength)) {
+    throw new Error("Embedding response vectors have inconsistent lengths");
+  }
+  return ordered;
+}
+var EMBED_BATCH_SIZE = 16;
+async function ensureTokenEmbeddings(db, opts) {
+  const endpoint = await resolveUsableEmbeddingEndpoint(db);
+  if (!endpoint) {
+    const reason = await describeUnavailableReason(db);
+    return { status: "unavailable", embedded: 0, remaining: 0, reason };
+  }
+  const model = canonicalEmbeddingModelId(endpoint.model);
+  const pending = await listTokensNeedingEmbedding(db, model, {
+    limit: opts?.limit ?? 64,
+    force: opts?.force,
+    dims: opts?.dims
+  });
+  let embedded = 0;
+  let failure;
+  try {
+    for (let i = 0; i < pending.length; i += EMBED_BATCH_SIZE) {
+      const batch = pending.slice(i, i + EMBED_BATCH_SIZE);
+      const formattedTexts = batch.map(
+        (item) => embeddingTextForToken(item.token, endpoint.model)
+      );
+      const vectors = await embedTexts(
+        { url: endpoint.url, model: endpoint.model, apiKey: endpoint.apiKey },
+        formattedTexts
+      );
+      for (const [index, item] of batch.entries()) {
+        await upsertTokenEmbedding(db, {
+          tokenId: item.token.id,
+          embedding: vectors[index],
+          model,
+          contentHash: computeContentHash(item.text)
+        });
+        embedded++;
+      }
+    }
+  } catch (err) {
+    failure = err instanceof Error ? err.message : String(err);
+  }
+  const coverage = await getEmbeddingCoverage(db, model, { dims: opts?.dims });
+  const remaining = coverage.missing + coverage.stale;
+  if (failure !== void 0) {
+    return { status: "unavailable", embedded, remaining, reason: failure };
+  }
+  return { status: "ok", embedded, remaining };
+}
+async function describeUnavailableReason(db) {
+  const cfg = await getProviderForRole(db, "embedding");
+  if (!cfg.enabled) {
+    return "embedding role is disabled in settings (llm.enabled)";
+  }
+  const endpoints = [cfg, ...cfg.fallback ? [cfg.fallback] : []];
+  const online = await Promise.all(
+    endpoints.map((endpoint) => isLlmOnline(endpoint.url))
+  );
+  if (!online.some(Boolean)) {
+    return `embedding endpoints offline (${endpoints.map((endpoint) => endpoint.url).join(", ")})`;
+  }
+  return `no configured embedding model is available (${endpoints.map((endpoint) => endpoint.model).join(", ")})`;
+}
+async function embedQuery(db, text) {
+  const endpoint = await resolveUsableEmbeddingEndpoint(db);
+  if (!endpoint) return null;
+  try {
+    const [vector] = await embedTexts(
+      { url: endpoint.url, model: endpoint.model, apiKey: endpoint.apiKey },
+      [embeddingTextForQuery(text, endpoint.model)]
+    );
+    return { vector, model: canonicalEmbeddingModelId(endpoint.model) };
+  } catch {
+    return null;
+  }
+}
+async function findPossibleDuplicates(db, candidate, embed = embedQuery) {
+  const queryText = embeddingContentForToken({
+    concept: candidate.concept,
+    question: candidate.question ?? null,
+    domain: candidate.domain ?? ""
+  });
+  const q2 = await embed(db, queryText);
+  if (!q2) {
+    return [];
+  }
+  const coverage = await getEmbeddingCoverage(db, q2.model, {
+    dims: q2.vector.length
+  });
+  const totalNeeding = coverage.missing + coverage.stale;
+  if (totalNeeding > 0) {
+    await ensureTokenEmbeddings(db, { limit: 100, dims: q2.vector.length });
+    if (totalNeeding > 100) {
+      console.warn(
+        `Warning: Duplicate check is incomplete. ${totalNeeding - 100} tokens are still missing embeddings. Run 'zam token reembed' to complete indexing.`
+      );
+    }
+  }
+  const thresholdStr = await getSetting(db, "search.dedup_threshold");
+  const parsed = thresholdStr ? Number.parseFloat(thresholdStr) : Number.NaN;
+  const threshold = Number.isFinite(parsed) && parsed > 0 && parsed <= 1 ? parsed : 0.85;
+  const hits = await searchTokensHybrid(db, queryText, {
+    queryEmbedding: q2.vector,
+    model: q2.model,
+    limit: 1e3,
+    vectorTopK: 1e3
+  });
+  const results = [];
+  for (const hit of hits) {
+    if (hit.similarity !== null && hit.similarity >= threshold) {
+      results.push({
+        slug: hit.slug,
+        concept: hit.concept,
+        similarity: hit.similarity
+      });
+    }
+  }
+  return results;
+}
+
 // src/cli/llm/vision.ts
 import { randomBytes } from "crypto";
 import { readFileSync as readFileSync10 } from "fs";
@@ -10549,7 +11020,7 @@ var VALID_API_FLAVORS = [
   "chat-completions",
   "anthropic-messages"
 ];
-var VALID_ROLES = ["vision", "recall", "text"];
+var VALID_ROLES = ["vision", "recall", "text", "embedding"];
 function upsertProviderRecord(providers, name, patch) {
   const merged = { ...providers[name] ?? {} };
   if (patch.url !== void 0) merged.url = patch.url;
@@ -11182,14 +11653,23 @@ async function backupDatabaseTo(db, targetDir) {
 
 // src/cli/commands/bridge.ts
 var isServeMode = false;
+var serveStdinPayload;
 function jsonOut2(data) {
   console.log(JSON.stringify(data, null, 2));
 }
 function jsonError(message) {
+  let msg = message;
+  if (message.startsWith('{"error":')) {
+    try {
+      const parsed = JSON.parse(message);
+      msg = parsed.error;
+    } catch {
+    }
+  }
   if (isServeMode) {
-    throw new Error(JSON.stringify({ error: message }));
+    throw new Error(JSON.stringify({ error: msg }));
   }
-  console.log(JSON.stringify({ error: message }, null, 2));
+  console.log(JSON.stringify({ error: msg }, null, 2));
   process.exit(1);
 }
 function parseNonNegativeIntegerOption(name, value) {
@@ -11512,7 +11992,10 @@ bridgeCommand.command("agent-open").description("Launch an agent harness in the
     });
   });
 });
-bridgeCommand.command("get-review").description("Get next review card with prompt (JSON)").option("--user <id>", "User ID (default: whoami)").option("--no-resolve", "Skip resolving the token's source_link into context").action(async (opts) => {
+bridgeCommand.command("get-review").description("Get next review card with prompt (JSON)").option("--user <id>", "User ID (default: whoami)").option("--no-resolve", "Skip resolving the token's source_link into context").option(
+  "--no-dynamic-question",
+  "Use the stored question without generating a fresh LLM question"
+).action(async (opts) => {
   await withDb2(async (db) => {
     const userId = await resolveUser(opts, db, { json: true });
     const queue = await buildReviewQueue(db, {
@@ -11536,7 +12019,7 @@ bridgeCommand.command("get-review").description("Get next review card with promp
     let resolvedQuestion = item.question;
     let questionSource = "original";
     let questionModel;
-    if (isLlmEnabled) {
+    if (isLlmEnabled && opts.dynamicQuestion !== false) {
       try {
         const healed = await ensureHighQualityQuestion(db, {
           id: item.tokenId,
@@ -11783,11 +12266,16 @@ bridgeCommand.command("analyze-monitor").description("Analyze monitor log with t
       });
       return;
     }
-    const chunks = [];
-    for await (const chunk of process.stdin) {
-      chunks.push(chunk);
+    let raw;
+    if (isServeMode) {
+      raw = serveStdinPayload ?? "";
+    } else {
+      const chunks = [];
+      for await (const chunk of process.stdin) {
+        chunks.push(chunk);
+      }
+      raw = Buffer.concat(chunks).toString("utf-8").trim();
     }
-    const raw = Buffer.concat(chunks).toString("utf-8").trim();
     if (!raw) {
       jsonError("No input received on stdin. Pipe JSON with token patterns.");
     }
@@ -11812,13 +12300,17 @@ bridgeCommand.command("analyze-monitor").description("Analyze monitor log with t
   }
 });
 bridgeCommand.command("add-token").description("Create a token + card from JSON stdin").option("--user <id>", "User ID (default: whoami)").action(async (opts) => {
-  let db;
-  try {
-    const chunks = [];
-    for await (const chunk of process.stdin) {
-      chunks.push(chunk);
+  await withDb2(async (db) => {
+    let raw;
+    if (isServeMode) {
+      raw = serveStdinPayload ?? "";
+    } else {
+      const chunks = [];
+      for await (const chunk of process.stdin) {
+        chunks.push(chunk);
+      }
+      raw = Buffer.concat(chunks).toString("utf-8").trim();
     }
-    const raw = Buffer.concat(chunks).toString("utf-8").trim();
     if (!raw) {
       jsonError("No input received on stdin. Pipe JSON with token data.");
     }
@@ -11831,8 +12323,12 @@ bridgeCommand.command("add-token").description("Create a token + card from JSON
     if (!data?.slug || !data?.concept) {
       jsonError("JSON must include 'slug' and 'concept' fields");
     }
-    db = await openDatabase();
     const userId = await resolveUser(opts, db, { json: true });
+    const possibleDuplicates = await findPossibleDuplicates(db, {
+      concept: data?.concept,
+      question: data?.question ?? null,
+      domain: data?.domain
+    });
     const token = await createToken(db, {
       slug: data?.slug,
       concept: data?.concept,
@@ -11844,6 +12340,10 @@ bridgeCommand.command("add-token").description("Create a token + card from JSON
       question: data?.question ?? null
     });
     const card = await ensureCard(db, token.id, userId);
+    try {
+      await ensureTokenEmbeddings(db, { limit: 8 });
+    } catch {
+    }
     jsonOut2({
       success: true,
       token,
@@ -11854,15 +12354,79 @@ bridgeCommand.command("add-token").description("Create a token + card from JSON
         state: card.state,
         dueAt: card.due_at,
         blocked: card.blocked
+      },
+      possible_duplicates: possibleDuplicates
+    });
+  });
+});
+bridgeCommand.command("relevant-tokens").description("Find tokens relevant to a given context").option("--user <id>", "User ID (default: whoami)").action(async (opts) => {
+  await withDb2(async (db) => {
+    let raw;
+    if (isServeMode) {
+      raw = serveStdinPayload ?? "";
+    } else {
+      const chunks = [];
+      for await (const chunk of process.stdin) {
+        chunks.push(chunk);
       }
+      raw = Buffer.concat(chunks).toString("utf-8").trim();
+    }
+    if (!raw) {
+      jsonError("No input received on stdin. Pipe JSON with context.");
+    }
+    let data;
+    try {
+      data = JSON.parse(raw);
+    } catch {
+      jsonError("Invalid JSON input");
+    }
+    if (!data?.context || data.context.trim() === "") {
+      jsonError("JSON must include a non-empty 'context' field");
+    }
+    const userId = await resolveUser(opts, db, { json: true });
+    const truncatedContext = data.context.slice(0, 2e3);
+    const q2 = await embedQuery(db, truncatedContext);
+    try {
+      await ensureTokenEmbeddings(db, {
+        limit: 32,
+        dims: q2?.vector.length
+      });
+    } catch {
+    }
+    let limit = data.limit ?? 10;
+    if (typeof limit !== "number" || limit <= 0 || !Number.isInteger(limit)) {
+      limit = 10;
+    }
+    if (limit > 100) {
+      limit = 100;
+    }
+    const results = await searchTokensHybrid(db, truncatedContext, {
+      queryEmbedding: q2?.vector,
+      model: q2?.model,
+      limit
     });
-    await db.close();
-  } catch (err) {
-    await db?.close();
-    if (err.message) {
-      jsonError(err.message);
+    const tokens = [];
+    for (const t2 of results) {
+      const card = await getCard(db, t2.id, userId);
+      tokens.push({
+        slug: t2.slug,
+        concept: t2.concept,
+        domain: t2.domain,
+        bloom_level: t2.bloom_level,
+        score: t2.score,
+        similarity: t2.similarity,
+        card: card ? {
+          state: card.state,
+          due_at: card.due_at,
+          blocked: card.blocked
+        } : null
+      });
     }
-  }
+    jsonOut2({
+      semantic: q2 !== null,
+      tokens
+    });
+  });
 });
 bridgeCommand.command("discover-skills").description(
   "Analyze monitor logs across sessions to discover recurring patterns"
@@ -12672,12 +13236,13 @@ bridgeCommand.command("check-llm").description("Check if LLM is enabled and onli
 });
 bridgeCommand.command("provider-status").description("Show secret-safe provider status for LLM roles (JSON)").action(async () => {
   await withDb2(async (db) => {
-    const [recall, vision, text] = await Promise.all([
+    const [recall, vision, text, embedding] = await Promise.all([
       getProviderRoleStatus(db, "recall"),
       getProviderRoleStatus(db, "vision"),
-      getProviderRoleStatus(db, "text")
+      getProviderRoleStatus(db, "text"),
+      getProviderRoleStatus(db, "embedding")
     ]);
-    jsonOut2({ roles: { recall, vision, text } });
+    jsonOut2({ roles: { recall, vision, text, embedding } });
   });
 });
 bridgeCommand.command("provider-config-list").description("List provider records and role bindings (JSON)").option("--scope <scope>", "machine (default) or shared", "machine").action(async (opts) => {
@@ -12983,6 +13548,46 @@ bridgeCommand.command("get-settings").description("Get active ZAM settings (JSON
     });
   });
 });
+async function readDatabaseUserSummaries(db) {
+  return await db.prepare(
+    `SELECT user_id AS id, COUNT(*) AS cardCount
+       FROM cards
+       GROUP BY user_id
+       ORDER BY user_id`
+  ).all();
+}
+bridgeCommand.command("database-status").description("Show the active database target and learning profiles (JSON)").action(async () => {
+  const target = getDatabaseTargetInfo();
+  await withDb2(async (db) => {
+    const userId = await getSetting(db, "user.id") ?? null;
+    const users = await readDatabaseUserSummaries(db);
+    jsonOut2({
+      success: true,
+      connected: true,
+      target,
+      userId,
+      cardCount: users.find((user) => user.id === userId)?.cardCount ?? 0,
+      users
+    });
+  });
+});
+bridgeCommand.command("database-select-user").description("Select an existing learning profile for this database (JSON)").requiredOption("--user <id>", "Existing user ID").action(async (opts) => {
+  await withDb2(async (db) => {
+    const userId = String(opts.user ?? "").trim();
+    const users = await readDatabaseUserSummaries(db);
+    const selected = users.find((user) => user.id === userId);
+    if (!selected) {
+      jsonError(`Learning profile not found: ${userId}`);
+      return;
+    }
+    await setSetting(db, "user.id", userId);
+    jsonOut2({
+      success: true,
+      userId,
+      cardCount: selected.cardCount
+    });
+  });
+});
 bridgeCommand.command("list-tokens").description(
   "List tokens (optionally enriched with user card state for viz) (JSON)"
 ).option(
@@ -13707,6 +14312,13 @@ bridgeCommand.command("serve").description("Start the persistent JSON-RPC stdin/
       requestId = req.id ?? null;
       const cmd = req.cmd;
       const args = req.args ?? [];
+      if (typeof req.stdin === "string") {
+        serveStdinPayload = req.stdin;
+      } else if (typeof req.stdin === "object" && req.stdin !== null) {
+        serveStdinPayload = JSON.stringify(req.stdin);
+      } else {
+        serveStdinPayload = void 0;
+      }
       if (!cmd) {
         return JSON.stringify({
           id: requestId,
@@ -13761,6 +14373,8 @@ bridgeCommand.command("serve").description("Start the persistent JSON-RPC stdin/
         id: requestId,
         error: `Invalid JSON request: ${err.message}`
       });
+    } finally {
+      serveStdinPayload = void 0;
     }
   };
   const readline = await import("readline");
@@ -16766,7 +17380,7 @@ import { Command as Command20 } from "commander";
 var tokenCommand = new Command20("token").description(
   "Manage knowledge tokens"
 );
-tokenCommand.command("register").description("Register a new knowledge token").requiredOption("--slug <slug>", "Unique token slug").requiredOption("--concept <concept>", "Concept description").option("--domain <domain>", "Knowledge domain", "").option("--bloom <level>", "Bloom taxonomy level (1-5)", "1").option("--source-link <link>", "Source file path or reference URL", "").option("--question <question>", "Specific question prompt for recall", "").option("--json", "Output as JSON").option("--quiet", "Suppress output (exit code only)").action(async (opts) => {
+tokenCommand.command("register").description("Register a new knowledge token").requiredOption("--slug <slug>", "Unique token slug").requiredOption("--concept <concept>", "Concept description").option("--domain <domain>", "Knowledge domain", "").option("--bloom <level>", "Bloom taxonomy level (1-5)", "1").option("--source-link <link>", "Source file path or reference URL", "").option("--question <question>", "Specific question prompt for recall", "").option("--user <id>", "Owner of the personal card (default: whoami)").option("--no-card", "Register the token only; do not create a personal card").option("--json", "Output as JSON").option("--quiet", "Suppress output (exit code only)").action(async (opts) => {
   await withDb(async (db) => {
     let question = opts.question || null;
     if (!question) {
@@ -16776,6 +17390,11 @@ tokenCommand.command("register").description("Register a new knowledge token").r
         opts.domain
       );
     }
+    const possibleDuplicates = await findPossibleDuplicates(db, {
+      concept: opts.concept,
+      question,
+      domain: opts.domain
+    });
     const token = await createToken(db, {
       slug: opts.slug,
       concept: opts.concept,
@@ -16784,9 +17403,30 @@ tokenCommand.command("register").description("Register a new knowledge token").r
       source_link: opts.sourceLink || null,
       question
     });
+    let cardUserId = null;
+    if (opts.card !== false) {
+      cardUserId = opts.user ?? await getSetting(db, "user.id");
+      if (cardUserId) {
+        await ensureCard(db, token.id, cardUserId);
+      }
+    }
+    try {
+      await ensureTokenEmbeddings(db, { limit: 8 });
+    } catch {
+    }
     if (opts.quiet) return;
     if (opts.json) {
-      console.log(JSON.stringify(token, null, 2));
+      console.log(
+        JSON.stringify(
+          {
+            ...token,
+            card: cardUserId ? { userId: cardUserId } : null,
+            possible_duplicates: possibleDuplicates
+          },
+          null,
+          2
+        )
+      );
     } else {
       console.log(`Registered token: ${token.slug} (${token.id})`);
       console.log(`  Concept:  ${token.concept}`);
@@ -16796,12 +17436,41 @@ tokenCommand.command("register").description("Register a new knowledge token").r
       if (token.source_link) {
         console.log(`  Source:   ${token.source_link}`);
       }
+      if (cardUserId) {
+        console.log(`  Card:     created for ${cardUserId}`);
+      } else if (opts.card === false) {
+        console.log(`  Card:     skipped (--no-card)`);
+      } else {
+        console.log(`  Card:     skipped (no default user set)`);
+      }
+      if (possibleDuplicates.length > 0) {
+        console.log(`
+WARNING: Possible duplicate tokens found:`);
+        for (const dup of possibleDuplicates) {
+          console.log(
+            `  - ${dup.slug} (similarity: ${dup.similarity.toFixed(2)})`
+          );
+        }
+      }
     }
   });
 });
 tokenCommand.command("find").description("Fuzzy search for tokens").requiredOption("--query <query>", "Search query").option("--json", "Output as JSON").option("--quiet", "Suppress output (exit code only)").action(async (opts) => {
   await withDb(async (db) => {
-    const results = await findTokens(db, opts.query);
+    const q2 = await embedQuery(db, opts.query);
+    const embRes = await ensureTokenEmbeddings(db, {
+      limit: 32,
+      dims: q2?.vector.length
+    });
+    if (embRes.status === "unavailable" && !opts.json && !opts.quiet) {
+      console.error(
+        `Note: semantic search unavailable (${embRes.reason}) \u2014 lexical matches only.`
+      );
+    }
+    const results = await searchTokensHybrid(db, opts.query, {
+      queryEmbedding: q2?.vector,
+      model: q2?.model
+    });
     if (opts.quiet) return;
     if (opts.json) {
       console.log(JSON.stringify(results, null, 2));
@@ -16814,12 +17483,14 @@ tokenCommand.command("find").description("Fuzzy search for tokens").requiredOpti
     console.log(`Found ${results.length} token(s):
 `);
     console.log(
-      "Score  Slug                  Concept                         Domain      Bloom"
+      "Score  Sim  Slug                  Concept                         Domain      Bloom"
     );
-    console.log("\u2500".repeat(90));
+    console.log("\u2500".repeat(95));
     for (const t2 of results) {
+      const scoreStr = t2.score.toFixed(3).padEnd(6);
+      const simStr = (t2.similarity?.toFixed(2) ?? "-").padEnd(4);
       console.log(
-        `${String(t2.score).padEnd(6)} ${t2.slug.padEnd(21)} ${t2.concept.slice(0, 31).padEnd(31)} ${(t2.domain || "-").padEnd(11)} ${t2.bloom_level}`
+        `${scoreStr} ${simStr} ${t2.slug.padEnd(21)} ${t2.concept.slice(0, 31).padEnd(31)} ${(t2.domain || "-").padEnd(11)} ${t2.bloom_level}`
       );
     }
   });
@@ -17047,6 +17718,56 @@ tokenCommand.command("status").description("Show full status of a token for a us
     }
   });
 });
+tokenCommand.command("reembed").description("Backfill or refresh semantic-search embeddings for tokens").option("--all", "Force re-embed every token, including already-fresh ones").option("--json", "Output as JSON").option("--quiet", "Suppress output (exit code only)").action(async (opts) => {
+  await withDb(async (db) => {
+    const probe = await embedQuery(db, "ZAM embedding dimension probe");
+    const model = probe?.model ?? null;
+    const dims = probe?.vector.length;
+    const before = model ? await getEmbeddingCoverage(db, model, { dims }) : { tokens: 0, embedded: 0, missing: 0, stale: 0 };
+    let embedded = 0;
+    let reason;
+    let force = Boolean(opts.all);
+    while (true) {
+      const result = await ensureTokenEmbeddings(
+        db,
+        force ? { force: true, limit: Number.MAX_SAFE_INTEGER, dims } : { dims }
+      );
+      force = false;
+      embedded += result.embedded;
+      if (result.status === "unavailable") {
+        reason = result.reason;
+        break;
+      }
+      if (result.remaining === 0 || result.embedded === 0) break;
+    }
+    if (reason !== void 0) {
+      if (opts.quiet) {
+        process.exit(1);
+      }
+      if (opts.json) {
+        jsonOut({ error: reason, embedded });
+      } else {
+        console.error(`Error: ${reason}`);
+        if (embedded > 0) {
+          console.error(
+            `(${embedded} token(s) were embedded before the failure)`
+          );
+        }
+      }
+      process.exit(1);
+    }
+    const after = await getEmbeddingCoverage(db, model, { dims });
+    if (opts.quiet) return;
+    const staleBefore = before.missing + before.stale;
+    if (opts.json) {
+      jsonOut({ embedded, model, before, after });
+      return;
+    }
+    console.log(
+      `Embedded ${embedded} tokens (${after.tokens} total, ${staleBefore} stale before) with ${model}`
+    );
+  });
+});
 
 // src/cli/commands/ui.ts
 import { spawn as spawn3, spawnSync } from "child_process";