thumbgate 1.35.0 → 1.37.1

package/scripts/rag-embedding-identity.js

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * RAG embedding identity — fail-closed model/dim/provider contract.
+ *
+ * Research source (2026-08-12): Super Data Science #1017 with Pete Johnson
+ * (MongoDB Field CTO of AI) — "The RAG Mistake Almost Every Team Is Making"
+ * https://www.youtube.com/watch?v=jAPGTVlNoD4
+ *
+ * Claim translated into enforcement:
+ *   - Embedding model choice is the underrated ROI lever (not just the LLM).
+ *   - Mixing embedding models / dimensions silently destroys retrieval quality.
+ *   - Matryoshka progressive disclosure (coarse → fine) is a valid cost/quality
+ *     tradeoff only when truncation is from a single compatible base model.
+ */
+
+const {
+  normalizeToMatryoshkaDimension,
+  buildMatryoshkaConfig,
+  EMBEDDING_QUALITY_THRESHOLDS,
+} = require('./matryoshka-embedding');
+
+function parseEmbeddingIdentity(fingerprint, dimensionHint) {
+  const raw = String(fingerprint || 'unknown').trim() || 'unknown';
+  const parts = raw.split(':').map((p) => p.trim()).filter(Boolean);
+  const dimensionFromParts = parts
+    .map((p) => Number(p))
+    .find((n) => Number.isFinite(n) && n > 0);
+  const dimension = Number.isFinite(Number(dimensionHint)) && Number(dimensionHint) > 0
+    ? Number(dimensionHint)
+    : dimensionFromParts || null;
+  const model = parts.find((p) => /embed|gemini|text-|qwen|openai|voyage|bge|e5|mini/i.test(p)) || parts[1] || null;
+  const source = parts[0] || raw;
+  return {
+    fingerprint: raw,
+    source,
+    model,
+    dimension,
+    identityKey: [source, model || 'unspecified', dimension || 'na'].join('|'),
+  };
+}
+
+function assertCompatibleEmbeddings({
+  queryProvider,
+  queryDimension,
+  documentProvider,
+  documentDimension,
+  allowUnknown = false,
+} = {}) {
+  const query = parseEmbeddingIdentity(queryProvider, queryDimension);
+  const document = parseEmbeddingIdentity(documentProvider, documentDimension);
+  const issues = [];
+
+  if (!allowUnknown && (query.fingerprint === 'unknown' || document.fingerprint === 'unknown')) {
+    issues.push({
+      code: 'unknown_embedding_identity',
+      severity: 'high',
+      message: 'Embedding provider fingerprint is unknown; refuse silent semantic ranking.',
+    });
+  }
+
+  if (
+    query.fingerprint
+    && document.fingerprint
+    && query.fingerprint !== 'unknown'
+    && document.fingerprint !== 'unknown'
+    && query.fingerprint !== document.fingerprint
+  ) {
+    // Same provider family with only dim suffix mismatch is handled below.
+    const queryBase = query.fingerprint.replace(/:\d+$/, '');
+    const docBase = document.fingerprint.replace(/:\d+$/, '');
+    if (queryBase !== docBase) {
+      issues.push({
+        code: 'embedding_provider_mismatch',
+        severity: 'high',
+        message: `Query provider ${query.fingerprint} is incompatible with document provider ${document.fingerprint}.`,
+      });
+    }
+  }
+
+  if (
+    Number.isFinite(query.dimension)
+    && Number.isFinite(document.dimension)
+    && query.dimension !== document.dimension
+  ) {
+    issues.push({
+      code: 'embedding_dimension_mismatch',
+      severity: 'high',
+      message: `Query dim ${query.dimension} != document dim ${document.dimension}. Use Matryoshka truncation from one base vector, not mixed spaces.`,
+    });
+  }
+
+  return {
+    ok: issues.length === 0,
+    query,
+    document,
+    issues,
+    failClosed: issues.some((i) => i.severity === 'high'),
+  };
+}
+
+/**
+ * Progressive Matryoshka plan (coarse filter → fine re-rank).
+ * Pete Johnson framing: cheaper dims for funnel; full dims for precision.
+ */
+function buildProgressiveRetrievalPlan(options = {}) {
+  const config = buildMatryoshkaConfig(options);
+  const baseDim = config.baseDimension;
+  const coarseDim = normalizeToMatryoshkaDimension(
+    Math.min(256, baseDim)
+  );
+  const midDim = normalizeToMatryoshkaDimension(
+    Math.min(768, baseDim)
+  );
+  return {
+    strategy: 'progressive_matryoshka',
+    researchSource: {
+      episode: 'Super Data Science #1017',
+      guest: 'Pete Johnson (MongoDB Field CTO of AI)',
+      url: 'https://www.youtube.com/watch?v=jAPGTVlNoD4',
+      thesis: 'Embedding model choice + compatible dimensions dominate RAG ROI.',
+    },
+    baseDimension: baseDim,
+    stages: [
+      {
+        stage: 1,
+        name: 'coarse_filter',
+        dimension: coarseDim,
+        topKMultiplier: 4,
+        purpose: 'cheap semantic funnel over full corpus',
+      },
+      {
+        stage: 2,
+        name: 'fine_rerank',
+        dimension: midDim < baseDim ? midDim : baseDim,
+        topKMultiplier: 1,
+        purpose: 'precision re-rank of coarse survivors',
+      },
+    ],
+    qualityThresholds: EMBEDDING_QUALITY_THRESHOLDS,
+    embeddingModel: config.embeddingModel,
+    provider: config.provider,
+    rules: [
+      'Never cosine-compare vectors from two different embedding models.',
+      'Truncate Matryoshka vectors only from a single full base embedding.',
+      'Re-embed the whole corpus when provider or base model changes.',
+      'Prefer hybrid lexical+dense until embedding identity is proven stable.',
+    ],
+  };
+}
+
+const MIN_GOLDEN_CASES = 6;
+
+function scoreEmbeddingRoiReadiness({
+  providerPinned = false,
+  modelPinned = false,
+  dimensionPinned = false,
+  hybridEnabled = false,
+  goldenRecall = null,
+  goldenPrecision = null,
+  goldenCaseCount = null,
+  perCaseRecalls = null,
+  mixedProviderCorpus = false,
+} = {}) {
+  const issues = [];
+  if (mixedProviderCorpus) {
+    issues.push('mixed_provider_corpus');
+  }
+  if (!providerPinned) issues.push('provider_unpinned');
+  if (!modelPinned) issues.push('model_unpinned');
+  if (!dimensionPinned) issues.push('dimension_unpinned');
+  if (!hybridEnabled) issues.push('hybrid_disabled');
+  if (!Number.isFinite(goldenRecall) || goldenRecall < EMBEDDING_QUALITY_THRESHOLDS.recall) {
+    issues.push('golden_recall_below_bar');
+  }
+  if (!Number.isFinite(goldenPrecision) || goldenPrecision < EMBEDDING_QUALITY_THRESHOLDS.precision) {
+    issues.push('golden_precision_below_bar');
+  }
+
+  // AGENTS.md production RAG gate: ≥6 golden cases and 100% per-case recall.
+  // Aggregate recall/precision alone must not mark ready (Codex P2).
+  const caseCount = Number.isFinite(Number(goldenCaseCount))
+    ? Number(goldenCaseCount)
+    : (Array.isArray(perCaseRecalls) ? perCaseRecalls.length : null);
+  if (!Number.isFinite(caseCount) || caseCount < MIN_GOLDEN_CASES) {
+    issues.push('golden_case_count_below_bar');
+  }
+  if (!Array.isArray(perCaseRecalls) || perCaseRecalls.length < MIN_GOLDEN_CASES) {
+    issues.push('per_case_recall_missing');
+  } else if (perCaseRecalls.some((r) => !Number.isFinite(Number(r)) || Number(r) < 1)) {
+    issues.push('per_case_recall_below_bar');
+  }
+
+  // ROI framing from episode: committees+metrics without embedding discipline = no ROI.
+  const score = Math.max(0, 100 - issues.length * 12);
+  return {
+    score,
+    ready: issues.length === 0,
+    issues,
+    minGoldenCases: MIN_GOLDEN_CASES,
+    recommendation: issues.includes('mixed_provider_corpus')
+      ? 'Rebuild the embedding cache under one provider+model+dim before trusting dense recall.'
+      : issues.includes('model_unpinned')
+        ? 'Pin THUMBGATE_EMBED_PROVIDER / model explicitly — embedding choice is the underrated ROI lever.'
+      : (issues.includes('golden_case_count_below_bar')
+          || issues.includes('per_case_recall_missing')
+          || issues.includes('per_case_recall_below_bar'))
+        ? 'Provide ≥6 golden cases with 100% per-case recall before marking embedding ROI ready.'
+        : issues.length
+          ? 'Close embedding identity gaps, then re-run eval:rag golden suite.'
+          : 'Embedding identity is pinned; keep progressive Matryoshka + hybrid eval green.',
+  };
+}
+
+module.exports = {
+  parseEmbeddingIdentity,
+  assertCompatibleEmbeddings,
+  buildProgressiveRetrievalPlan,
+  scoreEmbeddingRoiReadiness,
+};

package/scripts/rag-precision-guardrails.js

@@ -2,6 +2,7 @@
 'use strict';
 
 const { listGateTemplates } = require('./gate-templates');
+const { buildMatryoshkaConfig } = require('./matryoshka-embedding');
 
 const DOCUMENT_RAG_CATEGORY = 'Document RAG Safety';
 const PRECISION_TEMPLATE_IDS = new Set([
@@ -22,7 +23,42 @@ function toNumber(value) {
   return Number.isFinite(num) ? num : null;
 }
 
+function normalizeProvider(value) {
+  const text = String(value || '').trim().toLowerCase();
+  return text || null;
+}
+
 function normalizeOptions(options = {}) {
+  const baselineProvider = normalizeProvider(options['baseline-provider'] || options.baselineProvider);
+  const newProvider = normalizeProvider(options['new-provider'] || options.newProvider || options['embedding-provider']);
+  const baselineModel = normalizeProvider(options['baseline-model'] || options.baselineModel);
+  const newModel = normalizeProvider(options['new-model'] || options.newModel || options['embedding-model']);
+  const baselineDim = toNumber(options['baseline-dim'] || options['baseline-dimensions'] || options.baselineDim);
+  const newDim = toNumber(options['new-dim'] || options['new-dimensions'] || options['matryoshka-dim'] || options.newDim);
+  const providerChanged = Boolean(
+    baselineProvider && newProvider && baselineProvider !== newProvider
+  );
+  const modelChanged = Boolean(
+    baselineModel && newModel && baselineModel !== newModel
+  );
+  const dimReduced = (
+    baselineDim !== null
+    && newDim !== null
+    && newDim > 0
+    && baselineDim > 0
+    && newDim < baselineDim
+  );
+  const embeddingModelChange = normalizeBoolean(
+    options['embedding-model-change']
+    || options['provider-change']
+    || options['model-change']
+  ) || providerChanged || modelChanged;
+  const matryoshkaTruncation = normalizeBoolean(
+    options['matryoshka-truncation']
+    || options['matryoshka']
+    || options['dim-truncation']
+  ) || dimReduced;
+
   return {
     ragTool: String(options['rag-tool'] || options.tool || 'agentic-rag').trim() || 'agentic-rag',
     baselineRecall: toNumber(options['baseline-recall'] || options.recall),
@@ -32,6 +68,14 @@ function normalizeOptions(options = {}) {
     topK: toNumber(options['top-k'] || options.k),
     thresholdChanged: normalizeBoolean(options['threshold-change'] || options['threshold-changed']),
     embeddingFineTune: normalizeBoolean(options['embedding-finetune'] || options['embedding-fine-tune'] || options['fine-tune']),
+    embeddingModelChange,
+    matryoshkaTruncation,
+    baselineProvider,
+    newProvider,
+    baselineModel,
+    newModel,
+    baselineDim,
+    newDim,
     structuralNearMisses: normalizeBoolean(options['structural-near-misses'] || options['near-misses']),
     verifier: normalizeBoolean(options.verifier || options.reranker || options['second-stage']),
     hybridRetrieval: normalizeBoolean(options['hybrid-retrieval'] || options.hybrid),
@@ -57,6 +101,8 @@ function templateApplicability(template, options) {
   if (template.id === 'require-rag-baseline-before-precision-tuning') {
     return options.thresholdChanged ||
       options.embeddingFineTune ||
+      options.embeddingModelChange ||
+      options.matryoshkaTruncation ||
       options.baselineRecall === null ||
       options.newRecall === null ||
       (recallDropPercent(options) !== null && recallDropPercent(options) > 5);
@@ -75,6 +121,8 @@ function buildSignals(options) {
   const drop = recallDropPercent(options);
   return [
     precisionTuningSignal(options, drop),
+    embeddingModelSignal(options),
+    matryoshkaDimSignal(options),
     ragCascadeSignal(options),
     verifierLatencySignal(options),
     hybridScaleSignal(options),
@@ -83,19 +131,63 @@ function buildSignals(options) {
 }
 
 function precisionTuningSignal(options, drop) {
-  if (!(options.thresholdChanged || options.embeddingFineTune || drop !== null)) return null;
+  if (!(options.thresholdChanged || options.embeddingFineTune || options.embeddingModelChange
+    || options.matryoshkaTruncation || drop !== null)) return null;
   return {
     id: 'precision_tuning',
     label: 'Precision tuning change',
     values: [
       options.thresholdChanged ? 'threshold changed' : null,
       options.embeddingFineTune ? 'embedding fine-tune' : null,
+      options.embeddingModelChange ? 'embedding model/provider change' : null,
+      options.matryoshkaTruncation ? 'matryoshka dim truncation' : null,
       drop !== null ? `recall drop ${drop}%` : null,
     ].filter(Boolean),
     risk: 'precision wins can hide broad retrieval recall regressions',
   };
 }
 
+function embeddingModelSignal(options) {
+  if (!options.embeddingModelChange) return null;
+  const values = [
+    options.baselineProvider && options.newProvider
+      ? `provider ${options.baselineProvider} -> ${options.newProvider}`
+      : null,
+    options.baselineModel && options.newModel
+      ? `model ${options.baselineModel} -> ${options.newModel}`
+      : 'embedding model/provider swap proposed',
+    options.baselineRecall === null ? 'missing baseline recall@k' : null,
+    options.newRecall === null ? 'missing post-change recall@k' : null,
+  ].filter(Boolean);
+  return {
+    id: 'embedding_model_choice',
+    label: 'Embedding model choice risk',
+    values,
+    risk: 'embedding model choice is the highest-leverage RAG decision; swaps without a recall baseline are the most common silent regression',
+  };
+}
+
+function matryoshkaDimSignal(options) {
+  if (!options.matryoshkaTruncation) return null;
+  const values = [
+    options.baselineDim !== null && options.newDim !== null
+      ? `dims ${options.baselineDim} -> ${options.newDim}`
+      : 'matryoshka truncation proposed',
+    options.baselineDim !== null && options.newDim !== null && options.newDim < options.baselineDim
+      ? `storage cut ${Math.round((1 - (options.newDim / options.baselineDim)) * 100)}%`
+      : null,
+    options.baselineRecall === null || options.newRecall === null
+      ? 'truncation must re-measure recall@k after L2 re-normalization'
+      : null,
+  ].filter(Boolean);
+  return {
+    id: 'matryoshka_dim_truncation',
+    label: 'Matryoshka dimension truncation',
+    values,
+    risk: 'leading-dim truncation only stays quality-safe after re-normalization and a measured recall baseline — never cut dims as a silent cost optimization',
+  };
+}
+
 function ragCascadeSignal(options) {
   if (!(options.structuralNearMisses || options.agenticPipeline)) return null;
   return {
@@ -187,6 +279,14 @@ function buildRagPrecisionGuardrailsPlan(rawOptions = {}, templatesPath) {
       recallDropPercent: recallDropPercent(options),
       baselinePrecision: options.baselinePrecision,
       newPrecision: options.newPrecision,
+      embeddingModelChange: options.embeddingModelChange,
+      matryoshkaTruncation: options.matryoshkaTruncation,
+      baselineProvider: options.baselineProvider,
+      newProvider: options.newProvider,
+      baselineModel: options.baselineModel,
+      newModel: options.newModel,
+      baselineDim: options.baselineDim,
+      newDim: options.newDim,
       hybridRetrieval: options.hybridRetrieval,
       denseRetrieval: options.denseRetrieval,
       sparseRetrieval: options.sparseRetrieval,
@@ -205,15 +305,25 @@ function buildRagPrecisionGuardrailsPlan(rawOptions = {}, templatesPath) {
     },
     signals,
     templates,
+    matryoshka: options.matryoshkaTruncation
+      ? buildMatryoshkaConfig({
+        embeddingDim: options.newDim || options.baselineDim || 768,
+        embeddingModel: options.newModel || options.baselineModel || undefined,
+        provider: options.newProvider || options.baselineProvider || undefined,
+      })
+      : null,
     nextActions: [
       'Save baseline recall@k, precision@k, answer-with-evidence, and latency before tuning retrieval.',
+      'Treat embedding model choice as a first-class change: re-index, re-measure, and keep a rollback provider fingerprint.',
+      'When using Matryoshka/MRL truncation, re-L2-normalize truncated vectors and re-run the golden recall suite before shipping the smaller dim.',
       'Block embedding or threshold changes when recall drops without an approved rollback plan.',
       'Use a second-stage verifier or reranker for structural near misses such as negation and role reversal.',
       'Attach verifier latency budgets before routing the retrieval output into autonomous agent actions.',
       'Measure dense recall, sparse recall, reranked relevance, source grounding, ACL filtering, and freshness as separate production gates.',
       'Treat the retrieval layer as the agent ground truth: every autonomous action should carry source evidence and access-control proof.',
+      'Promote only structured agentic memory (lessons, rules, outcomes) — never raw transport transcripts or oversized chat dumps.',
     ],
-    exampleCommand: 'npx thumbgate rag-precision-guardrails --hybrid-retrieval --dense --sparse --scale-corpus-documents=1000000 --agentic --json',
+    exampleCommand: 'npx thumbgate rag-precision-guardrails --embedding-model-change --baseline-provider=nomic --new-provider=gemini --baseline-dim=768 --new-dim=256 --baseline-recall=0.95 --new-recall=0.91 --agentic --json',
   };
 }
 

package/scripts/remote-feedback-capture.js

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * remote-feedback-capture.js
+ *
+ * Hosted capture path for unattended/cloud agents when the Mac-local
+ * lessons store is missing. POSTs to THUMBGATE_API_BASE_URL/v1/feedback/capture
+ * with Bearer THUMBGATE_API_KEY.
+ */
+
+const path = require('node:path');
+
+const DEFAULT_PATH = '/v1/feedback/capture';
+
+function resolveBaseUrl(env = process.env) {
+  return String(env.THUMBGATE_API_BASE_URL || env.THUMBGATE_API_URL || '').trim().replace(/\/$/, '');
+}
+
+function resolveApiKey(env = process.env) {
+  return String(env.THUMBGATE_API_KEY || '').trim();
+}
+
+function isRemoteCaptureConfigured(env = process.env) {
+  return Boolean(resolveBaseUrl(env) && resolveApiKey(env));
+}
+
+/**
+ * @param {object} params
+ * @param {string} params.signal - up|down|positive|negative
+ * @param {string} params.context
+ * @param {string} [params.whatWentWrong]
+ * @param {string} [params.whatToChange]
+ * @param {string} [params.whatWorked]
+ * @param {string[]|string} [params.tags]
+ * @param {typeof fetch} [params.fetchImpl]
+ * @param {object} [params.env]
+ */
+async function captureFeedbackRemote(params = {}) {
+  const env = params.env || process.env;
+  const base = resolveBaseUrl(env);
+  const key = resolveApiKey(env);
+  if (!base || !key) {
+    return {
+      ok: false,
+      error: 'remote_capture_not_configured',
+      message: 'Set THUMBGATE_API_BASE_URL and THUMBGATE_API_KEY for hosted capture.',
+    };
+  }
+
+  const tags = Array.isArray(params.tags)
+    ? params.tags
+    : String(params.tags || '')
+      .split(',')
+      .map((t) => t.trim())
+      .filter(Boolean);
+
+  const body = {
+    signal: params.signal || params.feedback || 'down',
+    context: params.context || '',
+    whatWentWrong: params.whatWentWrong || null,
+    whatToChange: params.whatToChange || null,
+    whatWorked: params.whatWorked || null,
+    tags,
+    source: params.source || 'remote-feedback-capture',
+  };
+
+  const fetchImpl = params.fetchImpl || globalThis.fetch;
+  if (typeof fetchImpl !== 'function') {
+    return { ok: false, error: 'fetch_unavailable', message: 'global fetch is not available' };
+  }
+
+  const url = `${base}${DEFAULT_PATH}`;
+  try {
+    const response = await fetchImpl(url, {
+      method: 'POST',
+      headers: {
+        authorization: `Bearer ${key}`,
+        'content-type': 'application/json',
+        accept: 'application/json',
+      },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(params.timeoutMs || 20000),
+    });
+    const text = await response.text();
+    let json = null;
+    try {
+      json = text ? JSON.parse(text) : null;
+    } catch {
+      json = { raw: text };
+    }
+    if (!response.ok) {
+      return {
+        ok: false,
+        error: 'remote_capture_http_error',
+        status: response.status,
+        body: json,
+      };
+    }
+    return { ok: true, status: response.status, body: json };
+  } catch (err) {
+    return {
+      ok: false,
+      error: 'remote_capture_network_error',
+      message: err && err.message ? err.message : String(err),
+    };
+  }
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (const arg of argv) {
+    if (!arg.startsWith('--')) continue;
+    const [k, ...rest] = arg.slice(2).split('=');
+    out[k] = rest.length ? rest.join('=') : true;
+  }
+  return out;
+}
+
+async function main(argv = process.argv.slice(2)) {
+  const args = parseArgs(argv);
+  if (args.help) {
+    console.log('Usage: node scripts/remote-feedback-capture.js --feedback=down --context="..." [--what-went-wrong=...] [--tags=a,b]');
+    process.exit(0);
+  }
+  const result = await captureFeedbackRemote({
+    signal: args.feedback || args.signal || 'down',
+    context: args.context || '',
+    whatWentWrong: args['what-went-wrong'],
+    whatToChange: args['what-to-change'],
+    whatWorked: args['what-worked'],
+    tags: args.tags,
+  });
+  console.log(JSON.stringify(result, null, 2));
+  process.exit(result.ok ? 0 : 2);
+}
+
+if (pathResolveMain()) {
+  main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}
+
+function pathResolveMain() {
+  try {
+    return require('node:path').resolve(process.argv[1] || '') === require('node:path').resolve(__filename);
+  } catch {
+    return false;
+  }
+}
+
+module.exports = {
+  captureFeedbackRemote,
+  isRemoteCaptureConfigured,
+  parseArgs,
+  resolveBaseUrl,
+  resolveApiKey,
+};