thumbgate 1.34.3 → 1.37.1

package/scripts/lesson-embedding-index.js

@@ -153,6 +153,10 @@ function resolveProviderFingerprint(options, vector) {
  * Returns [{ id, score }] sorted descending. Embeds the query once; reuses cached
  * document vectors and only embeds new/changed lessons.
  *
+ * Fail-closed on embedding identity (Pete Johnson / SDS #1017): never cosine
+ * vectors from different providers or base dimensions. Optional progressive
+ * Matryoshka: coarse truncate filter, then fine re-rank survivors.
+ *
  * @returns {Promise<Array<{id:string, score:number}>>}
  */
 async function semanticRank(queryText, lessons = [], options = {}) {
@@ -162,6 +166,8 @@ async function semanticRank(queryText, lessons = [], options = {}) {
     persist = true,
     pruneCache = true,
     truncateDimension = null,
+    progressiveMatryoshka = false,
+    progressiveTopK = null,
     cacheFile = CACHE_FILE,
   } = options;
   if (!queryText || !Array.isArray(lessons) || lessons.length === 0) return [];
@@ -170,19 +176,36 @@ async function semanticRank(queryText, lessons = [], options = {}) {
   const cache = readCache(cachePath);
   let cacheDirty = false;
 
-  let queryVector = await embedder(queryText, { kind: 'query', task: 'code retrieval' });
-  if (!Array.isArray(queryVector) || queryVector.length === 0) return [];
-  
-  if (truncateDimension) {
-    queryVector = truncateVector(queryVector, truncateDimension);
-  }
-  const provider = resolveProviderFingerprint(options, queryVector);
-  const dimension = queryVector.length;
+  const fullQueryVector = await embedder(queryText, { kind: 'query', task: 'code retrieval' });
+  if (!Array.isArray(fullQueryVector) || fullQueryVector.length === 0) return [];
+
+  const provider = resolveProviderFingerprint(options, fullQueryVector);
+  const dimension = fullQueryVector.length;
   if (!options.embedder && /(?:built-in|feature-hash)/i.test(provider)) {
     throw new Error('Semantic embedding provider degraded to feature hashing');
   }
 
-  const scored = [];
+  const { assertCompatibleEmbeddings, buildProgressiveRetrievalPlan } = require('./rag-embedding-identity');
+
+  // Progressive Matryoshka: coarse funnel first, full cosine only for survivors.
+  // Never score full-dim then coarse-filter — that is strictly more expensive.
+  let progressivePlan = null;
+  let coarseDim = null;
+  if (progressiveMatryoshka && !truncateDimension) {
+    progressivePlan = buildProgressiveRetrievalPlan({ embeddingDim: dimension });
+    const plannedCoarse = progressivePlan.stages[0].dimension;
+    if (plannedCoarse < dimension) coarseDim = plannedCoarse;
+  }
+
+  const scoreDim = truncateDimension || null;
+  const compareQueryVector = scoreDim
+    ? truncateVector(fullQueryVector, scoreDim)
+    : fullQueryVector;
+  const coarseQueryVector = coarseDim
+    ? truncateVector(fullQueryVector, coarseDim)
+    : null;
+
+  const candidates = [];
   for (const lesson of lessons) {
     if (!lesson || !lesson.id) continue;
     const text = lessonText(lesson);
@@ -198,6 +221,7 @@ async function semanticRank(queryText, lessons = [], options = {}) {
       || !Array.isArray(entry.vector)
       || entry.vector.length !== dimension
     ) {
+      // Provider/dim drift → re-embed. Never cosine across mismatched identities.
       const vector = await embedder(text, {
         kind: 'document',
         task: 'code retrieval',
@@ -205,13 +229,32 @@ async function semanticRank(queryText, lessons = [], options = {}) {
       });
       if (!Array.isArray(vector) || vector.length === 0) continue;
       if (vector.length !== dimension) continue;
+      // Re-resolve AFTER each document embed so a same-dim fallback provider
+      // cannot be cached under the query provider fingerprint (Codex P1).
+      const documentProvider = resolveProviderFingerprint(options, vector);
+      const identity = assertCompatibleEmbeddings({
+        queryProvider: provider,
+        queryDimension: dimension,
+        documentProvider,
+        documentDimension: vector.length,
+      });
+      if (!identity.ok) continue;
+      // Cache under the query-session provider only when identities match.
       entry = { hash, provider, dimension, vector };
       cache[lesson.id] = entry;
       cacheDirty = true;
+    } else {
+      // Cached entry still must match the active query identity.
+      const identity = assertCompatibleEmbeddings({
+        queryProvider: provider,
+        queryDimension: dimension,
+        documentProvider: entry.provider,
+        documentDimension: entry.dimension,
+      });
+      if (!identity.ok) continue;
     }
 
-    const docVector = truncateDimension ? truncateVector(entry.vector, truncateDimension) : entry.vector;
-    scored.push({ id: lesson.id, score: cosineSimilarity(queryVector, docVector) });
+    candidates.push({ id: lesson.id, vector: entry.vector });
   }
 
   // Prune only when the caller supplied the complete corpus. Metadata-filtered
@@ -229,7 +272,47 @@ async function semanticRank(queryText, lessons = [], options = {}) {
 
   if (persist && cacheDirty) writeCache(cachePath, cache);
 
-  return scored.sort((a, b) => b.score - a.score);
+  if (candidates.length === 0) return [];
+
+  // Coarse-first path: score at coarse dim, then full-dim only survivors.
+  if (coarseDim && coarseQueryVector && candidates.length > 1) {
+    const coarseScored = candidates.map((row) => ({
+      id: row.id,
+      score: cosineSimilarity(coarseQueryVector, truncateVector(row.vector, coarseDim)),
+      vector: row.vector,
+    })).sort((a, b) => b.score - a.score);
+
+    const keepBase = Math.max(
+      1,
+      Number(progressiveTopK) || Math.min(candidates.length, Math.ceil(candidates.length / 2) || 8)
+    );
+    const keep = Math.min(
+      coarseScored.length,
+      keepBase * (progressivePlan.stages[0].topKMultiplier || 4)
+    );
+    const survivors = coarseScored.slice(0, keep);
+    return survivors
+      .map((row) => ({
+        id: row.id,
+        score: cosineSimilarity(fullQueryVector, row.vector),
+      }))
+      .sort((a, b) => b.score - a.score);
+  }
+
+  // Standard path: single-pass cosine at requested (or full) dimension.
+  return candidates
+    .map((row) => {
+      const docVector = scoreDim ? truncateVector(row.vector, scoreDim) : row.vector;
+      if (docVector.length !== compareQueryVector.length) {
+        return null;
+      }
+      return {
+        id: row.id,
+        score: cosineSimilarity(compareQueryVector, docVector),
+      };
+    })
+    .filter(Boolean)
+    .sort((a, b) => b.score - a.score);
 }
 
 module.exports = {

package/scripts/lesson-retrieval.js

@@ -14,6 +14,7 @@
  */
 
 const RECENCY_DECAY_DAYS = 30;
+const { DEFAULT_HALF_LIFE_MS, applyTemporalDecay } = require('./temporal-decay-weighting');
 const RERANK_CANDIDATE_POOL = 50; // bi-encoder retrieves this many; reranker picks topK
 const MAX_RETRIEVAL_MEMORY_CHARS = 20000;
 
@@ -173,6 +174,48 @@ async function buildQueryPlan(query, options = {}) {
   }
 }
 
+/**
+ * Measure retrieveRelevantLessons against a wall-clock budget.
+ * Concurrent agent load (New Stack retrieval-failure process, not a product):
+ * latency stacking is a budget, not "return everything". Over-budget is
+ * reported; the caller decides. Does not raise maxResults.
+ */
+function retrieveWithLatencyBudget(toolName, actionContext, options = {}) {
+  const started = Date.now();
+  const lessons = retrieveRelevantLessons(toolName, actionContext, options);
+  const latencyMs = Date.now() - started;
+  const budget = Number(options.latencyBudgetMs);
+  const hasBudget = Number.isFinite(budget) && budget > 0;
+  let oversizedRejected = false;
+  try {
+    const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
+    const pathMod = require('path');
+    const paths = options.feedbackDir
+      ? { MEMORY_LOG_PATH: pathMod.join(options.feedbackDir, 'memory-log.jsonl') }
+      : getFeedbackPaths();
+    const raw = readJSONL(paths.MEMORY_LOG_PATH, { maxLines: MAX_RETRIEVAL_MEMORY_LINES });
+    const maxChars = Number.isFinite(options.maxMemoryChars)
+      ? Math.max(1, options.maxMemoryChars)
+      : MAX_RETRIEVAL_MEMORY_CHARS;
+    oversizedRejected = raw.some((m) => {
+      if (!m || typeof m !== 'object') return false;
+      const combined = `${String(m.title || '')}\n${String(m.content || '')}`.trim();
+      return combined.length > maxChars;
+    });
+  } catch {
+    oversizedRejected = false;
+  }
+  return {
+    lessons,
+    count: Array.isArray(lessons) ? lessons.length : 0,
+    maxResults: options.maxResults == null ? 5 : options.maxResults,
+    latencyMs,
+    latencyBudgetMs: hasBudget ? budget : null,
+    overBudget: hasBudget ? latencyMs > budget : false,
+    oversizedRejected,
+  };
+}
+
 function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const { maxResults = 5, feedbackDir } = options;
 
@@ -228,21 +271,25 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
 
   const actionSig = buildActionSignature(toolName, actionContext);
 
-  // Stage 1 — local first-stage score, take top-50 candidates
-  const candidates = memories
+  // Stage 1 — local first-stage score, then a dedupe-aware pool cut. Collapsing
+  // near-duplicate clusters BEFORE the top-50 cut keeps a dense cluster from
+  // crowding genuinely distinct lessons out of the reranker's candidate pool.
+  const scored = memories
     .map((mem) => ({
       ...mem,
       relevanceScore: scoreRelevance(mem, toolName, actionContext, actionSig),
     }))
     .filter((m) => m.relevanceScore > 0.1)
-    .sort((a, b) => b.relevanceScore - a.relevanceScore)
-    .slice(0, RERANK_CANDIDATE_POOL);
+    .sort((a, b) => b.relevanceScore - a.relevanceScore);
+  const candidates = dedupeCandidatePool(scored);
 
   if (candidates.length === 0) return [];
 
-  // Stage 2 — field-aware BM25F reranker (not a neural cross-encoder)
+  // Stage 2 — field-aware BM25F reranker (not a neural cross-encoder). Over-fetch
+  // 2× so a post-rerank dedupe collapse backfills from ranked survivors instead of
+  // under-filling the caller's slot budget (mirrors the pragmatic path's topK).
   const reranked = rerankLessons(actionContext, candidates, {
-    topK: maxResults,
+    topK: Math.max(maxResults * 2, maxResults),
     toolName,
   });
 
@@ -251,7 +298,7 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const deduped = dedupeSupersededLessons(reranked);
   const selected = filterTopP(deduped, resolveTopP(options), { minKeep: options.minKeep });
 
-  const shaped = selected.map((m) => ({
+  const shaped = selected.slice(0, maxResults).map((m) => ({
     id: m.id,
     title: m.title,
     content: m.content,
@@ -434,8 +481,10 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
   if (conclusive) {
     // Short-circuit: skip embedding/dense search completely
     const { rerankLessons } = require('./lesson-reranker');
-    const reranked = rerankLessons(actionContext, lexicalScored.slice(0, RERANK_CANDIDATE_POOL), { topK: maxResults, toolName });
-    return filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
+    const reranked = rerankLessons(actionContext, dedupeCandidatePool(lexicalScored), { topK: Math.max(maxResults * 2, maxResults), toolName });
+    return filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep })
+      .slice(0, maxResults)
+      .map(shapeLesson);
   }
 
   const queryPlan = lexicalScored[0]?.relevanceScore >= (options.rewriteBelowScore ?? 0.6)
@@ -558,8 +607,8 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
   const lexById = new Map(lexicalScored.map((m) => [m.id, m.relevanceScore]));
   const topFusedScore = fused[0].score || 1;
 
-  const candidates = fused
-    .slice(0, RERANK_CANDIDATE_POOL)
+  const candidates = dedupeCandidatePool(fused
+    .slice(0, RERANK_CANDIDATE_POOL * 2)
     .map((entry) => {
       const mem = byId.get(entry.id);
       if (!mem) return null;
@@ -571,13 +620,15 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
         : entry.score / topFusedScore;
       return { ...mem, relevanceScore };
     })
-    .filter(Boolean);
+    .filter(Boolean));
 
   if (candidates.length === 0) return [];
 
   const { rerankLessons } = require('./lesson-reranker');
-  const reranked = rerankLessons(actionContext, candidates, { topK: maxResults, toolName });
-  const rows = filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
+  const reranked = rerankLessons(actionContext, candidates, { topK: Math.max(maxResults * 2, maxResults), toolName });
+  const rows = filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep })
+    .slice(0, maxResults)
+    .map(shapeLesson);
   return attachArrayRetrievalMeta(rows, {
     strategy: 'hybrid-rrf+bm25',
     indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
@@ -645,11 +696,26 @@ function scoreRelevance(memory, toolName, actionContext, actionSig) {
 
   if (memory.tags?.includes('negative')) score += 0.1;
 
-  if (memory.timestamp) {
-    const ageMs = Date.now() - new Date(memory.timestamp).getTime();
-    const ageDays = ageMs / (1000 * 60 * 60 * 24);
-    const decay = Math.max(0, 1 - ageDays / RECENCY_DECAY_DAYS);
-    score *= 0.5 + 0.5 * decay;
+  if (memory.timestamp || memory.receivedAt || memory.created_at) {
+    // Half-life temporal decay (scripts/temporal-decay-weighting.js), applied as
+    // a recency *factor* with the legacy 0.5 floor so old-but-still-valid
+    // lessons remain retrievable (pure exp decay zeroed fixtures in CI).
+    const stamp = memory.timestamp || memory.receivedAt || memory.created_at;
+    const halfLifeMs =
+      Number.isFinite(Number(process.env.THUMBGATE_RETRIEVAL_HALF_LIFE_MS)) &&
+      Number(process.env.THUMBGATE_RETRIEVAL_HALF_LIFE_MS) > 0
+        ? Number(process.env.THUMBGATE_RETRIEVAL_HALF_LIFE_MS)
+        : DEFAULT_HALF_LIFE_MS;
+    const activeMode = process.env.THUMBGATE_RETRIEVAL_ACTIVE_MODE === '1';
+    try {
+      const factor = applyTemporalDecay(1, stamp, halfLifeMs, activeMode);
+      score *= 0.5 + 0.5 * Math.max(0, Math.min(1, factor));
+    } catch {
+      const ageMs = Date.now() - new Date(stamp).getTime();
+      const ageDays = ageMs / (1000 * 60 * 60 * 24);
+      const decay = Math.max(0, 1 - ageDays / RECENCY_DECAY_DAYS);
+      score *= 0.5 + 0.5 * decay;
+    }
   }
 
   if (memory.structuredRule) score += 0.15;
@@ -791,6 +857,24 @@ function dedupeSupersededLessons(lessons, options = {}) {
   return kept.map((idx) => lessons[idx]);
 }
 
+/**
+ * Dedupe-aware candidate-pool cut. dedupeSupersededLessons() historically ran only
+ * AFTER the RERANK_CANDIDATE_POOL cut, so a dense cluster of near-duplicate lessons
+ * could fill the pool and crowd genuinely distinct lessons out before dedupe ever
+ * saw them. Collapse duplicates over a bounded window (pool × 2, keeping the O(n²)
+ * bigram pass cheap in the hook's hot path) BEFORE cutting to pool size. Clusters
+ * larger than the window can still crowd; store compaction is the durable fix.
+ *
+ * @param {Array<object>} scored - candidates sorted best-first
+ * @param {number} [pool=RERANK_CANDIDATE_POOL] - final pool size
+ * @returns {Array<object>} deduped best-first list, at most `pool` long
+ */
+function dedupeCandidatePool(scored, pool = RERANK_CANDIDATE_POOL) {
+  if (!Array.isArray(scored) || scored.length === 0) return [];
+  const bound = Math.max(1, Math.floor(pool));
+  return dedupeSupersededLessons(scored.slice(0, bound * 2)).slice(0, bound);
+}
+
 function calculateRetrievalEntropy(lessons) {
   if (!Array.isArray(lessons) || lessons.length === 0) return 0;
   let pW = 0, nW = 0, tW = 0;
@@ -807,6 +891,7 @@ function calculateRetrievalEntropy(lessons) {
 
 module.exports = {
   retrieveRelevantLessons,
+  retrieveWithLatencyBudget,
   retrieveRelevantLessonsAsync,
   reciprocalRankFusion,
   scoreRelevance,
@@ -817,6 +902,7 @@ module.exports = {
   filterTopP,
   resolveTopP,
   dedupeSupersededLessons,
+  dedupeCandidatePool,
   isRetrievableMemory,
   selectRetrievalMemories,
   matchesMetadataFilters,

package/scripts/local-model-profile.js

@@ -33,6 +33,16 @@ const GLM_MODEL_ROLES = {
   vlm: 'glm-4v-9b',
 };
 
+// vLLM / vllm.ai high-throughput model IDs for self-hosted local inference serving.
+// Activate by setting THUMBGATE_LOCAL_MODEL_FAMILY=vllm or THUMBGATE_LOCAL_LLM_ENDPOINT.
+const VLLM_MODEL_ROLES = {
+  normal: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+  thinking: 'deepseek-ai/DeepSeek-R1-Distill-Qwen-32B',
+  critique: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+  compaction: 'Qwen/Qwen2.5-Coder-7B-Instruct',
+  vlm: 'Qwen/Qwen2-VL-7B-Instruct',
+};
+
 const VALID_MODEL_ROLES = Object.keys(MODEL_ROLES);
 
 const EMBEDDING_PROFILES = {
@@ -342,9 +352,15 @@ function resolveModelRole(role, env) {
   const envKey = `THUMBGATE_MODEL_ROLE_${normalized.toUpperCase()}`;
   const modelFamily = resolveModelFamily(e);
   const isLocalGlm = modelFamily.startsWith('glm');
+  // Only explicit vLLM family/server selects VLLM_MODEL_ROLES. A generic
+  // THUMBGATE_LOCAL_LLM_ENDPOINT (Ollama, LM Studio, etc.) must keep its own model id.
+  const serverEngine = normalizeSlug(e.THUMBGATE_LOCAL_MODEL_SERVER || e.THUMBGATE_MODEL_SERVER || '');
+  const isLocalVllm = modelFamily.startsWith('vllm') || serverEngine === 'vllm';
   const providerMode = resolveProviderMode(e);
-  const provider = isLocalGlm ? 'local' : (providerMode === 'vertex' ? 'vertex' : 'gemini');
-  const defaultModel = isLocalGlm ? (GLM_MODEL_ROLES[normalized] || MODEL_ROLES[normalized]) : MODEL_ROLES[normalized];
+  const provider = (isLocalGlm || isLocalVllm) ? 'local' : (providerMode === 'vertex' ? 'vertex' : 'gemini');
+  const defaultModel = isLocalVllm
+    ? (VLLM_MODEL_ROLES[normalized] || MODEL_ROLES[normalized])
+    : (isLocalGlm ? (GLM_MODEL_ROLES[normalized] || MODEL_ROLES[normalized]) : MODEL_ROLES[normalized]);
   const model = (e[envKey] && String(e[envKey]).trim()) || defaultModel;
   return { role: normalized, model, provider, envKey };
 }
@@ -384,6 +400,7 @@ module.exports = {
   DEFAULT_FEEDBACK_DIR,
   EMBEDDING_PROFILES,
   GLM_MODEL_ROLES,
+  VLLM_MODEL_ROLES,
   INDEXCACHE_SERVER_ENGINES,
   LONG_CONTEXT_TAGS,
   LONG_CONTEXT_TASK_TYPES,

package/scripts/mailer/resend-mailer.js

@@ -19,7 +19,7 @@ const dns = require('node:dns').promises;
 
 const PRODUCT_NAME = 'ThumbGate Pro';
 const DASHBOARD_URL = 'https://thumbgate-production.up.railway.app/dashboard';
-const DEFAULT_CONTACT_EMAIL = 'igor.ganapolsky@gmail.com';
+const DEFAULT_CONTACT_EMAIL = 'support@thumbgate.ai';
 const DEFAULT_FROM = 'onboarding@resend.dev';
 const DEFAULT_REPLY_TO = DEFAULT_CONTACT_EMAIL;
 const DEFAULT_UNSUBSCRIBE_EMAIL = DEFAULT_CONTACT_EMAIL;

package/scripts/matryoshka-embedding.js

@@ -0,0 +1,235 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Matryoshka Embedding Implementation for ThumbGate
+ *
+ * Based on podcast insights from "The RAG Mistake Almost Every Team Is Making"
+ * Episode #1017, featuring Pete Johnson (Field CTO, AI at MongoDB)
+ *
+ * Matryoshka embeddings are nested representations where:
+ * - Lower dimensions contain meaningful semantic information at multiple granularities
+ * - Enables efficient retrieval without full-dimensional computation
+ * - Supports progressive disclosure: query at 256 dims, refine at 1024, full at 3072
+ */
+
+const PYRAMID_LAYERS = {
+  L0_CONVERSATION: 'L0_CONVERSATION',
+  L1_ATOM: 'L1_ATOM',
+  L2_SCENARIO: 'L2_SCENARIO',
+  L3_PERSONA_SOP: 'L3_PERSONA_SOP',
+};
+
+// Matryoshka dimension tiers - each tier contains the semantic information of lower tiers
+const MATRYOSHKA_DIMENSIONS = [256, 512, 768, 1024, 1536, 2048, 3072, 4096, 7680];
+
+// Embedding model quality thresholds for RAG pipelines
+const EMBEDDING_QUALITY_THRESHOLDS = {
+  recall: 0.95, // Minimum recall@k for baseline
+  precision: 0.15, // Minimum precision@k (covers 15% correctly)
+  cross_encoder_reranker: true, // Use cross-encoder for refinement
+};
+
+function normalizeToMatryoshkaDimension(requestedDim) {
+  const dim = Number(requestedDim) || 768;
+  if (dim <= 0 || !Number.isFinite(dim)) return 768;
+  let closest = MATRYOSHKA_DIMENSIONS[0];
+  let minDiff = Math.abs(dim - closest);
+  for (const tier of MATRYOSHKA_DIMENSIONS) {
+    const diff = Math.abs(tier - dim);
+    if (diff < minDiff) { closest = tier; minDiff = diff; }
+  }
+  return closest;
+}
+
+function getSemanticScore(dim) {
+  const tierIndex = MATRYOSHKA_DIMENSIONS.indexOf(dim);
+  if (tierIndex === -1) return 0;
+  return (tierIndex + 1) / MATRYOSHKA_DIMENSIONS.length;
+}
+
+function classifyMemoryLayer(memory = {}) {
+  const type = String(memory.type || 'episodic').toLowerCase();
+  const content = String(memory.content || '').toLowerCase();
+  const tags = Array.isArray(memory.tags)
+    ? new Set(memory.tags.map(t => String(t).toLowerCase()))
+    : new Set();
+
+  const transportLike = type === 'transcript'
+    || tags.has('transport')
+    || tags.has('transcript')
+    || /^(user|assistant|system|tool)\s*:/m.test(content);
+  if (transportLike) return PYRAMID_LAYERS.L0_CONVERSATION;
+
+  const explicitPolicy = /^(never|always|must|do not|require)\b/i.test(content.trim());
+  if (type === 'preference' || tags.has('sop') || tags.has('rule') || tags.has('policy')
+      || tags.has('guardrail') || explicitPolicy) {
+    return PYRAMID_LAYERS.L3_PERSONA_SOP;
+  }
+  if (type === 'procedural' || tags.has('workflow') || tags.has('scenario')
+      || tags.has('pipeline') || tags.has('playbook')) {
+    return PYRAMID_LAYERS.L2_SCENARIO;
+  }
+  if (type === 'semantic' || tags.has('fact') || tags.has('lesson') || tags.has('atom')
+      || memory.whatWentWrong || memory.whatWorked) {
+    return PYRAMID_LAYERS.L1_ATOM;
+  }
+  return PYRAMID_LAYERS.L0_CONVERSATION;
+}
+
+function buildEmbeddingTaskPrefix(layer, task = 'code retrieval') {
+  const layerWeights = {
+    [PYRAMID_LAYERS.L3_PERSONA_SOP]: 4,
+    [PYRAMID_LAYERS.L2_SCENARIO]: 3,
+    [PYRAMID_LAYERS.L1_ATOM]: 2,
+    [PYRAMID_LAYERS.L0_CONVERSATION]: 1,
+  };
+  const weight = layerWeights[layer] || 1;
+  const normalizedTask = String(task).trim().toLowerCase().replace(/[_-]+/g, ' ');
+  return `layer:${layer} weight:${weight} task:${normalizedTask} `;
+}
+
+function validateEmbeddingQuality({
+  recall,
+  precision,
+  recallBaseline,
+  precisionBaseline,
+  embeddingDim,
+  goldenCases,
+  perCaseRecall,
+} = {}) {
+  const issues = [];
+  if (!Number.isFinite(recall)) {
+    issues.push({ severity: 'high', issue: 'recall_missing_or_non_finite', actual: recall, recommended: 'Run the deterministic golden retrieval suite.' });
+  } else if (recall < EMBEDDING_QUALITY_THRESHOLDS.recall) {
+    issues.push({ severity: 'high', issue: 'recall_below_threshold', actual: recall, expected: EMBEDDING_QUALITY_THRESHOLDS.recall, recommended: 'Lower embedding dimension or use hybrid retrieval' });
+  }
+  if (!Number.isFinite(precision)) {
+    issues.push({ severity: 'high', issue: 'precision_missing_or_non_finite', actual: precision, recommended: 'Measure deterministic precision before promotion.' });
+  } else if (precision < EMBEDDING_QUALITY_THRESHOLDS.precision) {
+    issues.push({ severity: 'medium', issue: 'precision_below_threshold', actual: precision, expected: EMBEDDING_QUALITY_THRESHOLDS.precision, recommended: 'Enable cross-encoder reranker or increase dimension tier' });
+  }
+  if (!Number.isInteger(goldenCases) || goldenCases < 6) {
+    issues.push({ severity: 'high', issue: 'insufficient_golden_cases', actual: goldenCases, expected: 6, recommended: 'Provide at least six deterministic golden cases.' });
+  }
+  if (!Array.isArray(perCaseRecall) || perCaseRecall.length < 6
+      || perCaseRecall.some((value) => !Number.isFinite(value) || value < 1)) {
+    issues.push({ severity: 'high', issue: 'per_case_recall_incomplete', actual: perCaseRecall, expected: 'at least 6 cases at recall 1.0', recommended: 'Require 100% recall for every golden case.' });
+  }
+  if (Number.isFinite(recallBaseline) && Number.isFinite(precisionBaseline)
+      && Number.isFinite(recall) && Number.isFinite(precision)) {
+    if (recall < recallBaseline * 0.95) {
+      issues.push({ severity: 'high', issue: 'recall_regression_from_baseline', actual: recall, baseline: recallBaseline, recommended: 'Retrain embedding model with baseline preserved' });
+    }
+    if (precision < precisionBaseline * 0.95) {
+      issues.push({ severity: 'medium', issue: 'precision_regression_from_baseline', actual: precision, baseline: precisionBaseline, recommended: 'Verify embedding changes maintained retrieval quality' });
+    }
+  }
+  return {
+    valid: issues.length === 0,
+    issues,
+    dimensionTier: normalizeToMatryoshkaDimension(embeddingDim || 768),
+    semanticScore: getSemanticScore(normalizeToMatryoshkaDimension(embeddingDim || 768)),
+  };
+}
+
+function buildMatryoshkaConfig(options = {}) {
+  const baseDim = normalizeToMatryoshkaDimension(options.embeddingDim || 768);
+  const tiers = [];
+  let currentDim = 256;
+  while (currentDim <= baseDim) {
+    if (MATRYOSHKA_DIMENSIONS.includes(currentDim)) {
+      tiers.push({
+        dimension: currentDim,
+        semanticScore: getSemanticScore(currentDim),
+        useCase: currentDim < 512 ? 'fast_query' : currentDim < 1536 ? 'balanced' : 'full_precision',
+      });
+    }
+    currentDim += 256;
+  }
+  return {
+    baseDimension: baseDim,
+    tiers,
+    retrievalStrategy: options.retrievalStrategy || 'progressive_disclosure',
+    hybridRouting: options.hybridRouting !== undefined ? options.hybridRouting : true,
+    embeddingModel: options.embeddingModel || 'text-embedding-3-large',
+    provider: options.provider || 'openai',
+    qualityThresholds: EMBEDDING_QUALITY_THRESHOLDS,
+    rolloutPlan: [
+      'Use 256-dim embeddings for initial query filtering (fast, cheap)',
+      'Re-rank top candidates at 768-dim for balanced quality',
+      'Use full 1536+ dim embeddings for final precision or when cross-encoder confirms relevance',
+      'Log dimension-tier performance metrics for continuous optimization',
+    ],
+  };
+}
+
+function distillMemoryForEmbedding(memories = [], options = {}) {
+  const safeMemories = Array.isArray(memories) ? memories : [];
+  const layerGroups = {
+    [PYRAMID_LAYERS.L3_PERSONA_SOP]: [],
+    [PYRAMID_LAYERS.L2_SCENARIO]: [],
+    [PYRAMID_LAYERS.L1_ATOM]: [],
+    [PYRAMID_LAYERS.L0_CONVERSATION]: [],
+  };
+  const maxItemsPerLayer = options.maxItemsPerLayer || 100;
+  for (const memory of safeMemories) {
+    const layer = classifyMemoryLayer(memory);
+    if (layerGroups[layer].length < maxItemsPerLayer) {
+      layerGroups[layer].push({
+        ...memory,
+        pyramidLayer: layer,
+        semanticWeight: getSemanticScore(normalizeToMatryoshkaDimension(options.embeddingDim || 768)),
+      });
+    }
+  }
+  return {
+    kind: 'matryoshka-memory-distillation',
+    totalMemories: safeMemories.length,
+    layerDistribution: Object.entries(layerGroups).map(([layer, items]) => ({
+      layer,
+      count: items.length,
+      contentPreview: items.slice(0, 3).map(i => String(i.content || i.title || '').slice(0, 50)),
+    })),
+    layers: layerGroups,
+    embeddingStrategy: buildMatryoshkaConfig(options),
+  };
+}
+
+function generateEmbeddingPreventionRules(qualityReport) {
+  const rules = [];
+  if (!qualityReport.valid) {
+    for (const issue of qualityReport.issues) {
+      if (issue.issue === 'recall_below_threshold') {
+        const actualRecall = String(issue.actual).replace('.', '\\.');
+        rules.push({
+          id: 'block-low-embedding-recall',
+          name: 'Block embedding changes with low recall',
+          category: 'Embedding Quality',
+          signal: '👎',
+          defaultAction: 'block',
+          severity: 'high',
+          pattern: `(embedding|vector|retrieval).*(recall\\s*[:=]\\s*${actualRecall})`,
+          problem: 'Prevents deployment of embedding configurations that fail minimum recall thresholds.',
+          roi: 'Protects RAG pipeline effectiveness by ensuring embedding quality gates are maintained.',
+          rollout: 'Enable for all RAG workflows to prevent quality regressions.',
+        });
+      }
+    }
+  }
+  return rules;
+}
+
+module.exports = {
+  PYRAMID_LAYERS,
+  MATRYOSHKA_DIMENSIONS,
+  EMBEDDING_QUALITY_THRESHOLDS,
+  buildMatryoshkaConfig,
+  classifyMemoryLayer,
+  distillMemoryForEmbedding,
+  generateEmbeddingPreventionRules,
+  getSemanticScore,
+  normalizeToMatryoshkaDimension,
+  validateEmbeddingQuality,
+  buildEmbeddingTaskPrefix,
+};