zam-core 0.6.3 → 0.7.0

package/dist/index.d.ts

@@ -1010,6 +1010,97 @@ interface SourceProposalInput {
  */
 declare function confirmSourceImport(db: Database, userId: string, sourceId: string, proposals: SourceProposalInput[]): Promise<ConfirmFoundationsResult>;
 
+/**
+ * Token embedding repository — stores per-token vectors for semantic search
+ * (ADR 2026-07-03) and derives staleness from a content hash.
+ *
+ * This module is pure storage + classification: no HTTP, no LLM calls. The
+ * CLI layer (`src/cli/llm/embedder.ts`) generates vectors and calls in here.
+ */
+
+interface TokenEmbedding {
+    token_id: string;
+    model: string;
+    dims: number;
+    content_hash: string;
+    embedded_at: string;
+    embedding: Float32Array;
+}
+type EmbeddingStaleness = "missing" | "content-changed" | "model-changed" | "dimension-changed";
+interface TokenNeedingEmbedding {
+    token: Token;
+    /** Canonical text to embed — already hashed the same way. */
+    text: string;
+    reason: EmbeddingStaleness;
+}
+interface EmbeddingCoverage {
+    tokens: number;
+    embedded: number;
+    missing: number;
+    stale: number;
+}
+interface EmbeddedTokenRow {
+    token: Token;
+    embedding: Float32Array;
+}
+/**
+ * The canonical text embedded for a token. Every stored hash and every stored
+ * vector derives from exactly this string — never the slug, which is an
+ * identifier, not meaning.
+ */
+declare function embeddingContentForToken(t: Pick<Token, "concept" | "question" | "domain">): string;
+declare function computeContentHash(text: string): string;
+/**
+ * Encode a vector as a little-endian float32 BLOB. Builds a fresh buffer (no
+ * aliasing into the caller's array) so the row can be stored independently of
+ * whatever produced the vector.
+ */
+declare function encodeEmbedding(vec: ArrayLike<number>): Uint8Array;
+/**
+ * Decode a stored BLOB back into a Float32Array.
+ *
+ * BLOB values come back as `Buffer` (better-sqlite3) or `Uint8Array` (remote
+ * provider). better-sqlite3 Buffers are views into a shared pool and may have
+ * a non-zero, non-4-aligned `byteOffset` — constructing a Float32Array
+ * directly over such a buffer throws (RangeError) or silently reads garbage.
+ * Copying guarantees a fresh, 0-offset backing buffer — but `blob.slice()`
+ * cannot be used for this: `Buffer.prototype.slice` overrides
+ * `Uint8Array.prototype.slice` to return a *view* into the same backing
+ * buffer (a legacy Node.js Buffer API quirk), not a copy, so it would still
+ * carry the misaligned offset. Uint8Array's own `slice` must be borrowed
+ * explicitly to force an actual copy.
+ */
+declare function decodeEmbedding(blob: Uint8Array): Float32Array;
+declare function upsertTokenEmbedding(db: Database, input: {
+    tokenId: string;
+    embedding: ArrayLike<number>;
+    model: string;
+    contentHash: string;
+}): Promise<void>;
+declare function getTokenEmbedding(db: Database, tokenId: string): Promise<TokenEmbedding | undefined>;
+/**
+ * Classify every non-deprecated token against a target embedding model:
+ * missing (no stored row), model-changed (stored under a different model
+ * id), or content-changed (stored hash no longer matches the canonical
+ * text). `force: true` returns every token regardless of freshness, tagged
+ * `content-changed` since that is the closest-fitting reason to re-embed.
+ */
+declare function listTokensNeedingEmbedding(db: Database, model: string, opts?: {
+    limit?: number;
+    force?: boolean;
+    dims?: number;
+}): Promise<TokenNeedingEmbedding[]>;
+/** Same classification scan as {@link listTokensNeedingEmbedding}, counts only. */
+declare function getEmbeddingCoverage(db: Database, model: string, opts?: {
+    dims?: number;
+}): Promise<EmbeddingCoverage>;
+/**
+ * All tokens with a fresh vector under `model` to enter the vector search leg.
+ * Re-hashes rows here because lazy top-up is intentionally bounded: a stale
+ * row beyond the current batch must never participate with its old meaning.
+ */
+declare function listEmbeddedTokens(db: Database, model: string): Promise<EmbeddedTokenRow[]>;
+
 /**
  * Monitor log analyzer — maps observed shell commands to token ratings.
  *
@@ -1790,6 +1881,35 @@ interface ReviewQueue {
  */
 declare function buildReviewQueue(db: Database, options: ReviewQueueOptions): Promise<ReviewQueue>;
 
+/**
+ * Hybrid lexical/vector token search using reciprocal-rank fusion (RRF).
+ * (ADR 2026-07-03)
+ *
+ * This module is pure math + database queries — zero LLM dependencies, no HTTP.
+ */
+
+interface HybridSearchOptions {
+    queryEmbedding?: ArrayLike<number>;
+    /** Model the stored vectors must match; required when queryEmbedding is set. */
+    model?: string;
+    limit?: number;
+    rrfK?: number;
+    vectorTopK?: number;
+}
+interface HybridScoredToken extends Token {
+    score: number;
+    lexicalRank: number | null;
+    vectorRank: number | null;
+    similarity: number | null;
+}
+/** Calculates the cosine similarity between two float vectors. Returns 0 if either norm is 0. */
+declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
+/**
+ * Performs a hybrid search over active tokens.
+ * Combines lexical relevance rankings with vector cosine similarities.
+ */
+declare function searchTokensHybrid(db: Database, query: string, opts?: HybridSearchOptions): Promise<HybridScoredToken[]>;
+
 /**
  * Get the path to ZAM's internal package SKILL.md.
  */
@@ -1914,7 +2034,7 @@ interface InstallConfig {
     /** Machine-local id of the workspace currently active in this install. */
     activeWorkspaceId?: string;
 }
-type MachineAiRole = "vision" | "recall" | "text";
+type MachineAiRole = "vision" | "recall" | "text" | "embedding";
 type MachineApiFlavor = "chat-completions" | "anthropic-messages";
 interface MachineProviderRecord {
     label?: string;
@@ -2074,4 +2194,4 @@ declare function resolveAllBeliefPaths(db: Database): Promise<string[]>;
  */
 declare function resolveAllGoalPaths(db: Database): Promise<string[]>;
 
-export { type ADOConfig, type ADOCredentials, type AgentSkill, type AnalysisResult, type ApplySessionSynthesisInput, type ApplySessionSynthesisResult, BUILT_IN_SENSITIVE_MATCHERS, type BloomLevel$1 as BloomLevel, type CaptureDecision, type CaptureDenialReason, type CaptureRequest, type Card, type CardDeletionImpact, type CardState$1 as CardState, type CascadeBlockResult, type CommandRecord, type CommandSequence, type ConfirmFoundationsResult, type ConnectionOptions, type CreateAgentSkillInput, type CreateGoalInput, type CreateReviewInput, type CreateSessionInput, type CreateTokenInput, type Credentials, type CurriculumCardInput, DEFAULT_OBSERVER_POLICY, DEFAULT_REVIEW_CONTEXT_MAX_CHARS, type Database, type DatabaseProvider, type DatabaseTargetInfo, type DatabaseValue, type DeleteCardResult, type DeleteTokenResult, type DiscoveryOptions, type DomainCompetence, type EvaluateInput, type EvaluateResult, type ExecuteReviewActionInput, type ExecutionContext, type FSRSParameters, type FoundationProposalInput, type Goal, type GoalFrontmatter, type GoalStatus, type GoalSummary, HOMEBREW_CASK, type ImportCurriculumResult, type ImportResult, type InstallChannel, type InstallConfig, type InstallMode, type InstallPlan, type InstallResult, type LocalLLMRunner, type LogStepInput, type MachineAiConfig, type MachineProviderRecord, type MachineRoleBinding, type MonitorEvent, type Neighborhood, type NeighborhoodToken, OBSERVER_POLICY_UNSET_HINT, OBSERVER_POLICY_VERSION, type ObservationRating, type ObserverConsent, type ObserverPolicy, type ObserverRetention, type ObserverScope, type ObserverSettingKey, type OllamaDetectionOptions, type PersonalCard, type PrepareSessionSynthesisInput, type Prerequisite, type PrerequisiteWithToken, type PromptInput, REVIEW_CONTEXT_CACHE_TTL_MS, type Rating, type RecallPrompt, type RemoteDatabaseOptions, type RepoPaths, type ResolvedCaptureTarget, type ResolvedReference, type ReviewActionResult, type ReviewActionType, type ReviewContext, type ReviewLog, type ReviewQueue, type ReviewQueueItem, type ReviewQueueOptions, type RunResult, SIDECAR_POLICY_FILE, SNAPSHOT_VERSION, type SchedulingCard, type Session, type SessionStep, type SessionSummary, type SessionSynthesisCandidate, type SessionSynthesisEvidence, type SessionSynthesisPreview, type SessionSynthesisRecord, type SidecarPrivacyPolicy, type SkillProposal, type SkillSource, type SnapshotManifest, type SourceProposalInput, type SplitProposalInput, type Statement, type SupportedLocale, type SymbiosisMode, type SynthesisConfidence, type SystemProfile, type Token, type TokenDeleteImpact, type TokenPattern, type TranslationKey, type TursoCredentials, UI_OBSERVATION_PROTOCOL_VERSION, type UiActionType, type UiApplicationContext, type UiCandidateToken, type UiEvidenceRef, type UiEvidenceType, type UiObservationKind, type UiObservationReport, type UiObservedAction, type UnblockResult, type UpdateActionKind, type UpdateCardInput, type UpdateDecision, type UpdateStep, type UpdateStepKind, type UpdateTokenInput, type UserSetting, type UserStats, WINGET_PACKAGE_ID, type WorkItem, type WorkspaceConfig, type WorkspaceKind, type WorkspaceSourceControl, addPrerequisite, analyzeObservation, appendUiObservationReport, applySessionSynthesis, buildReviewQueue, buildUiSynthesisCandidates, cascadeBlock, clearADOCredentials, clearProviderApiKey, clearReviewContextCache, clearTursoCredentials, compareVersions, confirmCardSplit, confirmFoundations, confirmSourceImport, createAgentSkill, createFSRS, createGoal, createToken, decidePostCapture, decidePreCapture, decideUpdate, deleteCardForUser, deleteSetting, deleteToken, deprecateToken, detectSyncProvider, detectSystemLocale, discoverSkills, distributeGlobalSkills, endSession, ensureCard, ensureMonitorDir, ensureUiObserverDir, evaluateRating, executeReviewAction, exportSnapshot, extractTasks, extractTokenRefs, fetchActiveWorkItems, findTokens, generateBashHooks, generateBashUnhooks, generateConceptFreeCue, generatePowerShellHooks, generatePowerShellUnhooks, generatePrompt, generateTokenSlug, generateZshHooks, generateZshUnhooks, getADOCredentials, getActiveWorkspace, getActiveWorkspaceId, getAgentSkill, getAllSettings, getAllSettingsDetailed, getBlockedCards, getCard, getCardById, getCardDeletionImpact, getConfiguredWorkspaces, getDatabaseTargetInfo, getDefaultDbPath, getDependents, getDomainCompetence, getDueCards, getGoal, getGoalTree, getInstallChannel, getInstallMode, getMachineAiConfig, getMonitorDir, getMonitorLogStats, getMonitorPath, getPackageSkillPath, getPrerequisites, getProviderApiKey, getRepoPaths, getReviewsForCard, getReviewsForUser, getSessionSummary, getSessionSynthesisRecords, getSetting, getSystemProfile, getTokenById, getTokenBySlug, getTokenDeleteImpact, getTokenNeighborhood, getTursoCredentials, getUiObservationPath, getUiObserverDir, getUserStats, hasCommand, importCurriculumCards, importSnapshot, injectShellHooks, installFastFlowLM, installOllama, installOpenCode, interleave, isObserverPolicyConfigured, isOllamaInstalled, isUiObservationReport, listAgentSkills, listGoals, listPersonalCards, listProviderApiKeyRefs, listTokens, loadADOConfig, loadCredentials, loadInstallConfig, logReview, logStep, matchBuiltInSensitive, matchDenylist, matchesFilePath, monitorLogExists, normalizeLocale, normalizePath, openDatabase, openDatabaseWithSync, openRemoteDatabase, pairCommands, parseGoalFile, parseMonitorLog, parseObserverList, parseObserverPolicy, parseSnapshot, parseUiObservationLog, planOpenCodeInstall, planUpdate, prepareLocalModel, prepareSessionSynthesis, readMonitorLog, readUiObservationLog, removeConfiguredWorkspace, resolveAllBeliefPaths, resolveAllGoalPaths, resolveObserverPolicy, resolveOllamaCommand, resolveReference, resolveRepoPath, resolveReviewContext, saveConfiguredWorkspaces, saveCredentials, saveInstallConfig, saveMachineAiConfig, serializeGoal, setADOCredentials, setActiveWorkspaceId, setInstallChannel, setInstallMode, setProviderApiKey, setSetting, setTursoCredentials, slugify, startSession, syncObserverSidecarPolicy, t, toSidecarPrivacyPolicy, uiObservationLogExists, uiObservationTimeSpan, unblockReady, updateCard, updateGoalStatus, updateToken, upsertConfiguredWorkspace, verifySnapshot, wouldCreateCycle, writeMonitorEvent };
+export { type ADOConfig, type ADOCredentials, type AgentSkill, type AnalysisResult, type ApplySessionSynthesisInput, type ApplySessionSynthesisResult, BUILT_IN_SENSITIVE_MATCHERS, type BloomLevel$1 as BloomLevel, type CaptureDecision, type CaptureDenialReason, type CaptureRequest, type Card, type CardDeletionImpact, type CardState$1 as CardState, type CascadeBlockResult, type CommandRecord, type CommandSequence, type ConfirmFoundationsResult, type ConnectionOptions, type CreateAgentSkillInput, type CreateGoalInput, type CreateReviewInput, type CreateSessionInput, type CreateTokenInput, type Credentials, type CurriculumCardInput, DEFAULT_OBSERVER_POLICY, DEFAULT_REVIEW_CONTEXT_MAX_CHARS, type Database, type DatabaseProvider, type DatabaseTargetInfo, type DatabaseValue, type DeleteCardResult, type DeleteTokenResult, type DiscoveryOptions, type DomainCompetence, type EmbeddedTokenRow, type EmbeddingCoverage, type EmbeddingStaleness, type EvaluateInput, type EvaluateResult, type ExecuteReviewActionInput, type ExecutionContext, type FSRSParameters, type FoundationProposalInput, type Goal, type GoalFrontmatter, type GoalStatus, type GoalSummary, HOMEBREW_CASK, type HybridScoredToken, type HybridSearchOptions, type ImportCurriculumResult, type ImportResult, type InstallChannel, type InstallConfig, type InstallMode, type InstallPlan, type InstallResult, type LocalLLMRunner, type LogStepInput, type MachineAiConfig, type MachineProviderRecord, type MachineRoleBinding, type MonitorEvent, type Neighborhood, type NeighborhoodToken, OBSERVER_POLICY_UNSET_HINT, OBSERVER_POLICY_VERSION, type ObservationRating, type ObserverConsent, type ObserverPolicy, type ObserverRetention, type ObserverScope, type ObserverSettingKey, type OllamaDetectionOptions, type PersonalCard, type PrepareSessionSynthesisInput, type Prerequisite, type PrerequisiteWithToken, type PromptInput, REVIEW_CONTEXT_CACHE_TTL_MS, type Rating, type RecallPrompt, type RemoteDatabaseOptions, type RepoPaths, type ResolvedCaptureTarget, type ResolvedReference, type ReviewActionResult, type ReviewActionType, type ReviewContext, type ReviewLog, type ReviewQueue, type ReviewQueueItem, type ReviewQueueOptions, type RunResult, SIDECAR_POLICY_FILE, SNAPSHOT_VERSION, type SchedulingCard, type Session, type SessionStep, type SessionSummary, type SessionSynthesisCandidate, type SessionSynthesisEvidence, type SessionSynthesisPreview, type SessionSynthesisRecord, type SidecarPrivacyPolicy, type SkillProposal, type SkillSource, type SnapshotManifest, type SourceProposalInput, type SplitProposalInput, type Statement, type SupportedLocale, type SymbiosisMode, type SynthesisConfidence, type SystemProfile, type Token, type TokenDeleteImpact, type TokenEmbedding, type TokenNeedingEmbedding, type TokenPattern, type TranslationKey, type TursoCredentials, UI_OBSERVATION_PROTOCOL_VERSION, type UiActionType, type UiApplicationContext, type UiCandidateToken, type UiEvidenceRef, type UiEvidenceType, type UiObservationKind, type UiObservationReport, type UiObservedAction, type UnblockResult, type UpdateActionKind, type UpdateCardInput, type UpdateDecision, type UpdateStep, type UpdateStepKind, type UpdateTokenInput, type UserSetting, type UserStats, WINGET_PACKAGE_ID, type WorkItem, type WorkspaceConfig, type WorkspaceKind, type WorkspaceSourceControl, addPrerequisite, analyzeObservation, appendUiObservationReport, applySessionSynthesis, buildReviewQueue, buildUiSynthesisCandidates, cascadeBlock, clearADOCredentials, clearProviderApiKey, clearReviewContextCache, clearTursoCredentials, compareVersions, computeContentHash, confirmCardSplit, confirmFoundations, confirmSourceImport, cosineSimilarity, createAgentSkill, createFSRS, createGoal, createToken, decidePostCapture, decidePreCapture, decideUpdate, decodeEmbedding, deleteCardForUser, deleteSetting, deleteToken, deprecateToken, detectSyncProvider, detectSystemLocale, discoverSkills, distributeGlobalSkills, embeddingContentForToken, encodeEmbedding, endSession, ensureCard, ensureMonitorDir, ensureUiObserverDir, evaluateRating, executeReviewAction, exportSnapshot, extractTasks, extractTokenRefs, fetchActiveWorkItems, findTokens, generateBashHooks, generateBashUnhooks, generateConceptFreeCue, generatePowerShellHooks, generatePowerShellUnhooks, generatePrompt, generateTokenSlug, generateZshHooks, generateZshUnhooks, getADOCredentials, getActiveWorkspace, getActiveWorkspaceId, getAgentSkill, getAllSettings, getAllSettingsDetailed, getBlockedCards, getCard, getCardById, getCardDeletionImpact, getConfiguredWorkspaces, getDatabaseTargetInfo, getDefaultDbPath, getDependents, getDomainCompetence, getDueCards, getEmbeddingCoverage, getGoal, getGoalTree, getInstallChannel, getInstallMode, getMachineAiConfig, getMonitorDir, getMonitorLogStats, getMonitorPath, getPackageSkillPath, getPrerequisites, getProviderApiKey, getRepoPaths, getReviewsForCard, getReviewsForUser, getSessionSummary, getSessionSynthesisRecords, getSetting, getSystemProfile, getTokenById, getTokenBySlug, getTokenDeleteImpact, getTokenEmbedding, getTokenNeighborhood, getTursoCredentials, getUiObservationPath, getUiObserverDir, getUserStats, hasCommand, importCurriculumCards, importSnapshot, injectShellHooks, installFastFlowLM, installOllama, installOpenCode, interleave, isObserverPolicyConfigured, isOllamaInstalled, isUiObservationReport, listAgentSkills, listEmbeddedTokens, listGoals, listPersonalCards, listProviderApiKeyRefs, listTokens, listTokensNeedingEmbedding, loadADOConfig, loadCredentials, loadInstallConfig, logReview, logStep, matchBuiltInSensitive, matchDenylist, matchesFilePath, monitorLogExists, normalizeLocale, normalizePath, openDatabase, openDatabaseWithSync, openRemoteDatabase, pairCommands, parseGoalFile, parseMonitorLog, parseObserverList, parseObserverPolicy, parseSnapshot, parseUiObservationLog, planOpenCodeInstall, planUpdate, prepareLocalModel, prepareSessionSynthesis, readMonitorLog, readUiObservationLog, removeConfiguredWorkspace, resolveAllBeliefPaths, resolveAllGoalPaths, resolveObserverPolicy, resolveOllamaCommand, resolveReference, resolveRepoPath, resolveReviewContext, saveConfiguredWorkspaces, saveCredentials, saveInstallConfig, saveMachineAiConfig, searchTokensHybrid, serializeGoal, setADOCredentials, setActiveWorkspaceId, setInstallChannel, setInstallMode, setProviderApiKey, setSetting, setTursoCredentials, slugify, startSession, syncObserverSidecarPolicy, t, toSidecarPrivacyPolicy, uiObservationLogExists, uiObservationTimeSpan, unblockReady, updateCard, updateGoalStatus, updateToken, upsertConfiguredWorkspace, upsertTokenEmbedding, verifySnapshot, wouldCreateCycle, writeMonitorEvent };

package/dist/index.js

@@ -643,6 +643,18 @@ CREATE TABLE IF NOT EXISTS user_config (
   updated_at  TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
+-- Token embeddings: one vector per token for semantic search (ADR 2026-07-03).
+-- Not a column on tokens: hot paths do SELECT * FROM tokens and a ~3KB blob
+-- per row would ride along on all of them.
+CREATE TABLE IF NOT EXISTS token_embeddings (
+  token_id     TEXT PRIMARY KEY REFERENCES tokens(id) ON DELETE CASCADE,
+  embedding    BLOB NOT NULL,
+  model        TEXT NOT NULL,
+  dims         INTEGER NOT NULL,
+  content_hash TEXT NOT NULL,
+  embedded_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
 -- Agent skills: task recipes the agent learns from user guidance
 CREATE TABLE IF NOT EXISTS agent_skills (
   id          TEXT PRIMARY KEY,
@@ -989,6 +1001,16 @@ async function runMigrations(db) {
       await db.exec(`ALTER TABLE tokens ADD COLUMN topic_id TEXT`);
     }
   }
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS token_embeddings (
+      token_id     TEXT PRIMARY KEY REFERENCES tokens(id) ON DELETE CASCADE,
+      embedding    BLOB NOT NULL,
+      model        TEXT NOT NULL,
+      dims         INTEGER NOT NULL,
+      content_hash TEXT NOT NULL,
+      embedded_at  TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
 }
 
 // src/kernel/db/snapshot.ts
@@ -2451,6 +2473,165 @@ async function deleteSetting(db, key) {
   return result.changes > 0;
 }
 
+// src/kernel/models/token-embedding.ts
+import { createHash as createHash2 } from "crypto";
+function embeddingContentForToken(t2) {
+  return `${t2.concept}
+${t2.question ?? ""}
+${t2.domain}`;
+}
+function computeContentHash(text) {
+  return createHash2("sha256").update(text, "utf8").digest("hex");
+}
+function encodeEmbedding(vec) {
+  const f = Float32Array.from(vec);
+  return new Uint8Array(f.buffer);
+}
+function decodeEmbedding(blob) {
+  if (blob.byteLength % 4 !== 0) {
+    throw new Error(
+      "Invalid embedding blob size: must be a multiple of 4 bytes"
+    );
+  }
+  if (blob.byteOffset % 4 === 0) {
+    return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+  }
+  const copy = Uint8Array.prototype.slice.call(blob);
+  return new Float32Array(copy.buffer, 0, copy.byteLength / 4);
+}
+function decodeRow(row) {
+  return {
+    token_id: row.token_id,
+    model: row.model,
+    dims: row.dims,
+    content_hash: row.content_hash,
+    embedded_at: row.embedded_at,
+    embedding: decodeEmbedding(row.embedding)
+  };
+}
+async function upsertTokenEmbedding(db, input) {
+  const encoded = encodeEmbedding(input.embedding);
+  const dims = input.embedding.length;
+  const embeddedAt = (/* @__PURE__ */ new Date()).toISOString();
+  await db.prepare(`
+      INSERT INTO token_embeddings (token_id, embedding, model, dims, content_hash, embedded_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT(token_id) DO UPDATE SET
+        embedding = excluded.embedding,
+        model = excluded.model,
+        dims = excluded.dims,
+        content_hash = excluded.content_hash,
+        embedded_at = excluded.embedded_at
+    `).run(
+    input.tokenId,
+    encoded,
+    input.model,
+    dims,
+    input.contentHash,
+    embeddedAt
+  );
+}
+async function getTokenEmbedding(db, tokenId) {
+  const row = await db.prepare("SELECT * FROM token_embeddings WHERE token_id = ?").get(tokenId);
+  return row ? decodeRow(row) : void 0;
+}
+async function listTokensNeedingEmbedding(db, model, opts) {
+  const rows = await db.prepare(`
+      SELECT t.*, e.model AS emb_model, e.dims AS emb_dims, e.content_hash AS emb_hash
+      FROM tokens t
+      LEFT JOIN token_embeddings e ON e.token_id = t.id
+      WHERE t.deprecated_at IS NULL
+    `).all();
+  const needing = [];
+  for (const row of rows) {
+    let reason = null;
+    let text = "";
+    if (opts?.force) {
+      reason = "content-changed";
+    } else if (row.emb_model === null) {
+      reason = "missing";
+    } else if (row.emb_model !== model) {
+      reason = "model-changed";
+    } else if (opts?.dims !== void 0 && row.emb_dims !== opts.dims) {
+      reason = "dimension-changed";
+    } else {
+      const computedText = embeddingContentForToken(row);
+      const hash = computeContentHash(computedText);
+      if (row.emb_hash !== hash) {
+        reason = "content-changed";
+        text = computedText;
+      }
+    }
+    if (reason) {
+      if (!text) {
+        text = embeddingContentForToken(row);
+      }
+      needing.push({ token: row, text, reason });
+    }
+  }
+  if (opts?.limit !== void 0) {
+    return needing.slice(0, opts.limit);
+  }
+  return needing;
+}
+async function getEmbeddingCoverage(db, model, opts) {
+  const rows = await db.prepare(`
+      SELECT t.*, e.model AS emb_model, e.dims AS emb_dims, e.content_hash AS emb_hash
+      FROM tokens t
+      LEFT JOIN token_embeddings e ON e.token_id = t.id
+      WHERE t.deprecated_at IS NULL
+    `).all();
+  let missing = 0;
+  let stale = 0;
+  for (const row of rows) {
+    if (row.emb_model === null) {
+      missing++;
+      continue;
+    }
+    if (row.emb_model !== model) {
+      stale++;
+      continue;
+    }
+    if (opts?.dims !== void 0 && row.emb_dims !== opts.dims) {
+      stale++;
+      continue;
+    }
+    const hash = computeContentHash(embeddingContentForToken(row));
+    if (row.emb_hash !== hash) {
+      stale++;
+    }
+  }
+  const tokens = rows.length;
+  return {
+    tokens,
+    embedded: tokens - missing - stale,
+    missing,
+    stale
+  };
+}
+async function listEmbeddedTokens(db, model) {
+  const rows = await db.prepare(`
+      SELECT t.*, e.embedding AS embedding, e.content_hash AS emb_hash
+      FROM token_embeddings e
+      JOIN tokens t ON t.id = e.token_id
+      WHERE e.model = ? AND t.deprecated_at IS NULL
+    `).all(model);
+  return rows.flatMap((row) => {
+    const { embedding, emb_hash, ...token } = row;
+    if (emb_hash !== computeContentHash(embeddingContentForToken(token))) {
+      return [];
+    }
+    try {
+      return [{ token, embedding: decodeEmbedding(embedding) }];
+    } catch (err) {
+      console.warn(
+        `Warning: Corrupted embedding for token ${token.slug} (${token.id}) ignored: ${err.message}`
+      );
+      return [];
+    }
+  });
+}
+
 // src/kernel/observation/analyzer.ts
 function parseMonitorLog(jsonl) {
   const events = [];
@@ -4593,6 +4774,82 @@ function intersperseNew(reviews, newCards, interval) {
   return result;
 }
 
+// src/kernel/search/hybrid.ts
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  const len = a.length;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+async function searchTokensHybrid(db, query, opts) {
+  const limit = opts?.limit ?? 20;
+  const rrfK = opts?.rrfK ?? 60;
+  const vectorTopK = opts?.vectorTopK ?? 10;
+  const lexicalHits = await findTokens(db, query);
+  let vectorHits = [];
+  if (opts?.queryEmbedding && opts?.model) {
+    const queryVec = Float32Array.from(opts.queryEmbedding);
+    const embedded = await listEmbeddedTokens(db, opts.model);
+    const candidates = [];
+    for (const row of embedded) {
+      if (row.embedding.length !== queryVec.length) {
+        continue;
+      }
+      const similarity = cosineSimilarity(queryVec, row.embedding);
+      if (similarity <= 0) {
+        continue;
+      }
+      candidates.push({ token: row.token, similarity });
+    }
+    candidates.sort((a, b) => b.similarity - a.similarity);
+    vectorHits = candidates.slice(0, vectorTopK);
+  }
+  const tokenMap = /* @__PURE__ */ new Map();
+  const getOrCreateEntry = (token) => {
+    let entry = tokenMap.get(token.id);
+    if (!entry) {
+      entry = {
+        ...token,
+        score: 0,
+        lexicalRank: null,
+        vectorRank: null,
+        similarity: null
+      };
+      tokenMap.set(token.id, entry);
+    }
+    return entry;
+  };
+  for (let i = 0; i < lexicalHits.length; i++) {
+    const hit = lexicalHits[i];
+    const entry = getOrCreateEntry(hit);
+    entry.lexicalRank = i + 1;
+    entry.score += 1 / (rrfK + entry.lexicalRank);
+  }
+  for (let j = 0; j < vectorHits.length; j++) {
+    const hit = vectorHits[j];
+    const entry = getOrCreateEntry(hit.token);
+    entry.vectorRank = j + 1;
+    entry.similarity = hit.similarity;
+    entry.score += 1 / (rrfK + entry.vectorRank);
+  }
+  const results = Array.from(tokenMap.values());
+  results.sort((a, b) => {
+    if (Math.abs(a.score - b.score) > 1e-9) {
+      return b.score - a.score;
+    }
+    return a.slug.localeCompare(b.slug);
+  });
+  return results.slice(0, limit);
+}
+
 // src/kernel/system/hooks.ts
 import {
   appendFileSync as appendFileSync3,
@@ -5625,9 +5882,11 @@ export {
   clearReviewContextCache,
   clearTursoCredentials,
   compareVersions,
+  computeContentHash,
   confirmCardSplit,
   confirmFoundations,
   confirmSourceImport,
+  cosineSimilarity,
   createAgentSkill,
   createFSRS,
   createGoal,
@@ -5635,6 +5894,7 @@ export {
   decidePostCapture,
   decidePreCapture,
   decideUpdate,
+  decodeEmbedding,
   deleteCardForUser,
   deleteSetting,
   deleteToken,
@@ -5643,6 +5903,8 @@ export {
   detectSystemLocale,
   discoverSkills,
   distributeGlobalSkills,
+  embeddingContentForToken,
+  encodeEmbedding,
   endSession,
   ensureCard,
   ensureMonitorDir,
@@ -5679,6 +5941,7 @@ export {
   getDependents,
   getDomainCompetence,
   getDueCards,
+  getEmbeddingCoverage,
   getGoal,
   getGoalTree,
   getInstallChannel,
@@ -5700,6 +5963,7 @@ export {
   getTokenById,
   getTokenBySlug,
   getTokenDeleteImpact,
+  getTokenEmbedding,
   getTokenNeighborhood,
   getTursoCredentials,
   getUiObservationPath,
@@ -5717,10 +5981,12 @@ export {
   isOllamaInstalled,
   isUiObservationReport,
   listAgentSkills,
+  listEmbeddedTokens,
   listGoals,
   listPersonalCards,
   listProviderApiKeyRefs,
   listTokens,
+  listTokensNeedingEmbedding,
   loadADOConfig,
   loadCredentials,
   loadInstallConfig,
@@ -5760,6 +6026,7 @@ export {
   saveCredentials,
   saveInstallConfig,
   saveMachineAiConfig,
+  searchTokensHybrid,
   serializeGoal,
   setADOCredentials,
   setActiveWorkspaceId,
@@ -5780,6 +6047,7 @@ export {
   updateGoalStatus,
   updateToken,
   upsertConfiguredWorkspace,
+  upsertTokenEmbedding,
   verifySnapshot,
   wouldCreateCycle,
   writeMonitorEvent