thumbgate 1.29.2 → 1.31.0

package/scripts/lesson-search.js

@@ -4,6 +4,7 @@ const path = require('node:path');
 const { readJSONL, getFeedbackPaths } = require('./feedback-loop');
 const { buildMemoryLifecycleView, scoreHybridMemoryMatch } = require('./agent-memory-lifecycle');
 const { loadOptionalModule } = require('./private-core-boundary');
+const { selectRetrievalMemories } = require('./lesson-retrieval');
 
 const HIGH_RISK_TAGS = new Set([
   'billing',
@@ -402,6 +403,8 @@ function scoreLesson(queryText, memory, parsed, sourceFeedback) {
   if (!queryText) {
     return {
       score: recencyScore(memory.timestamp),
+      evidenceScore: 0,
+      priorScore: recencyScore(memory.timestamp),
       matchedTokens: [],
     };
   }
@@ -409,14 +412,16 @@ function scoreLesson(queryText, memory, parsed, sourceFeedback) {
   const lessonText = buildLessonQuery(memory, parsed, sourceFeedback);
   const queryTokens = tokenize(queryText);
   const lessonTokens = tokenize(lessonText);
-  const score = jaccardSimilarity(queryTokens, lessonTokens)
-    + substringBoost(queryText, lessonText)
-    + recencyScore(memory.timestamp)
+  const evidenceScore = jaccardSimilarity(queryTokens, lessonTokens)
+    + substringBoost(queryText, lessonText);
+  const priorScore = recencyScore(memory.timestamp)
     + (memory.category === 'error' ? 0.05 : 0)
     + Math.min(0.2, scoreHybridMemoryMatch(queryText, memory).score * 0.1);
 
   return {
-    score,
+    score: evidenceScore + priorScore,
+    evidenceScore,
+    priorScore,
     matchedTokens: unique(queryTokens.filter((token) => lessonTokens.includes(token))),
   };
 }
@@ -426,7 +431,7 @@ function buildLessonResult(memory, sourceFeedback, options = {}) {
   const lessonQuery = buildLessonQuery(memory, parsed, sourceFeedback);
   const ruleMatches = readPreventionRuleMatches(lessonQuery, Number(options.ruleLimit || 3), options);
   const gateMatches = buildGateMatches(memory, parsed, Number(options.gateLimit || 3), options);
-  const { score, matchedTokens } = scoreLesson(options.query || '', memory, parsed, sourceFeedback);
+  const { score, evidenceScore, priorScore, matchedTokens } = scoreLesson(options.query || '', memory, parsed, sourceFeedback);
   const harnessRecommendations = buildHarnessRecommendations(memory, parsed, sourceFeedback, ruleMatches, gateMatches);
   const lifecycle = buildLifecycle(memory, parsed, sourceFeedback, ruleMatches, gateMatches, harnessRecommendations);
   const memoryLifecycle = buildMemoryLifecycleView(memory, { query: options.query || '' });
@@ -440,6 +445,8 @@ function buildLessonResult(memory, sourceFeedback, options = {}) {
     timestamp: memory.timestamp || null,
     sourceFeedbackId: memory.sourceFeedbackId || null,
     score: Number(score.toFixed(4)),
+    evidenceScore: Number(evidenceScore.toFixed(4)),
+    priorScore: Number(priorScore.toFixed(4)),
     matchedTokens,
     lesson: {
       summary: parsed.summary,
@@ -481,7 +488,8 @@ function searchLessons(query = '', options = {}) {
   const sqliteResults = tryFts5Search(query, options);
   if (sqliteResults) return sqliteResults;
 
-  const memories = readJSONL(MEMORY_LOG_PATH);
+  const allMemories = readJSONL(MEMORY_LOG_PATH);
+  const memories = selectRetrievalMemories(allMemories, options);
   const feedbackEntries = readJSONL(FEEDBACK_LOG_PATH);
   const feedbackById = new Map(feedbackEntries.map((entry) => [entry.id, entry]));
   const parsedLimit = Number(options.limit || 10);
@@ -508,7 +516,10 @@ function searchLessons(query = '', options = {}) {
     results = results.filter((entry) => requiredTags.every((tag) => entry.tags.includes(tag)));
   }
   if (query) {
-    results = results.filter((entry) => entry.score > 0);
+    // Recency, error severity, and lifecycle priors can reorder candidates,
+    // but they are never evidence that a lesson answers the query. Returning a
+    // recent unrelated incident is worse than an honest empty result.
+    results = results.filter((entry) => entry.evidenceScore > 0);
   }
 
   results.sort((a, b) => {
@@ -516,7 +527,7 @@ function searchLessons(query = '', options = {}) {
     return String(b.timestamp || '').localeCompare(String(a.timestamp || ''));
   });
 
-  // Cross-encoder reranking: when a query is present, rerank the top-50 bi-encoder
+  // Field-aware BM25F reranking: when a query is present, rerank the top-50 first-stage
   // candidates using field-weighted BM25 so the most relevant lessons surface first.
   if (query && results.length > 1) {
     const { rerankLessons } = loadOptionalModule('./lesson-reranker', () => ({
@@ -534,9 +545,12 @@ function searchLessons(query = '', options = {}) {
     filters: {
       category: category || null,
       tags: requiredTags,
+      scope: options.scope || null,
+      requireScope: options.requireScope === true,
     },
     feedbackDir: FEEDBACK_DIR,
     totalLessons: memories.length,
+    excludedLessons: allMemories.length - memories.length,
     returned: Math.min(limit, results.length),
     results: results.slice(0, limit),
     backend: 'jsonl-jaccard',
@@ -548,10 +562,15 @@ function searchLessons(query = '', options = {}) {
  * or not opted in. Set LESSON_DB_SEARCH=1 to enable FTS5 as primary backend.
  */
 function tryFts5Search(query, options) {
+  // The SQLite index does not currently carry the complete four-field scope
+  // contract. Fall back to JSONL whenever isolation is requested rather than
+  // silently searching across tenants or sessions.
+  if (options.scope || options.requireScope) return null;
   if (!process.env.LESSON_DB_SEARCH && !options.useFts5) return null;
+  let db = null;
   try {
     const { initDB, searchLessons: fts5Search, getStats } = require('./lesson-db');
-    const db = initDB();
+    db = initDB();
     const stats = getStats(db);
 
     // If DB is empty, skip (not yet backfilled)
@@ -567,11 +586,24 @@ function tryFts5Search(query, options) {
         .map((tag) => tag.trim())
         .filter(Boolean);
 
-    const rows = fts5Search(db, query || '', {
-      limit,
+    const candidateRows = fts5Search(db, query || '', {
+      limit: Math.max(limit * 5, 50),
       signal,
       tags: requiredTags.length > 0 ? requiredTags : undefined,
     });
+    const retrievableRows = selectRetrievalMemories(
+      candidateRows.map((row) => ({
+        ...row,
+        title: row.context || '',
+        content: [
+          row.whatWentWrong,
+          row.whatToChange,
+          row.whatWorked,
+        ].filter(Boolean).join('\n'),
+      })),
+      options,
+    );
+    const rows = retrievableRows.slice(0, limit);
 
     return {
       query: String(query || ''),
@@ -581,6 +613,7 @@ function tryFts5Search(query, options) {
         tags: requiredTags,
       },
       totalLessons: stats.total,
+      excludedLessons: candidateRows.length - retrievableRows.length,
       returned: rows.length,
       results: rows.map((row) => ({
         id: row.id,
@@ -606,6 +639,10 @@ function tryFts5Search(query, options) {
     };
   } catch (_err) {
     return null; // SQLite unavailable — fall through to JSONL
+  } finally {
+    if (db) {
+      try { db.close(); } catch { /* best-effort close */ }
+    }
   }
 }
 

package/scripts/llm-client.js

@@ -14,12 +14,51 @@ const DEFAULT_MAX_TOKENS = 1024;
 const DEFAULT_CACHE_TTL = '5m';
 const DEFAULT_ZAI_BASE_URL = 'https://api.z.ai/api/paas/v4';
 const DEFAULT_ZAI_MODEL = 'glm-5.2-flash';
+const DEFAULT_GATEWAY_MODEL = 'glm-5.2';
+const GATEWAY_TIMEOUT_MS = 30000;
 
 let _anthropicClient = null;
 let _geminiClient = null;
 
-function isAvailable() {
-  return Boolean(process.env.ANTHROPIC_API_KEY);
+function getGatewayConfig(env = process.env) {
+  const baseUrl = String(env.THUMBGATE_LLM_GATEWAY_URL || '').trim();
+  if (!baseUrl) return null;
+  return {
+    baseUrl: baseUrl.replace(/\/+$/, ''),
+    model: String(env.THUMBGATE_LLM_GATEWAY_MODEL || DEFAULT_GATEWAY_MODEL).trim(),
+    credentialEnvVar: 'THUMBGATE_LLM_GATEWAY_TOKEN',
+  };
+}
+
+function isGatewayConfigured(env = process.env) {
+  return Boolean(getGatewayConfig(env));
+}
+
+function resolveGatewayModel(options = {}, config = getGatewayConfig()) {
+  const explicitGatewayModel = String(options.gatewayModel || '').trim();
+  if (explicitGatewayModel) return explicitGatewayModel;
+  return String(config?.model || DEFAULT_GATEWAY_MODEL).trim();
+}
+
+function describeInferenceAvailability(env = process.env) {
+  if (env.ANTHROPIC_API_KEY) return { available: true, provider: 'anthropic' };
+  const gateway = getGatewayConfig(env);
+  if (gateway) {
+    return {
+      available: true,
+      provider: 'gateway',
+      model: gateway.model,
+    };
+  }
+  return {
+    available: false,
+    provider: 'none',
+    reason: 'no ANTHROPIC_API_KEY and no THUMBGATE_LLM_GATEWAY_URL',
+  };
+}
+
+function isAvailable(env = process.env) {
+  return describeInferenceAvailability(env).available;
 }
 
 function getClient() {
@@ -155,6 +194,144 @@ function buildSafeProviderError(error) {
   return summary;
 }
 
+function normalizeUsageTelemetry(usage = null) {
+  if (!usage || typeof usage !== 'object') {
+    return {
+      inputTokens: null,
+      outputTokens: null,
+      cacheReadInputTokens: null,
+      cacheWriteInputTokens: null,
+    };
+  }
+  const finiteOrNull = (...values) => {
+    const value = values.map(Number).find(Number.isFinite);
+    return value === undefined ? null : value;
+  };
+  return {
+    inputTokens: finiteOrNull(usage.input_tokens, usage.prompt_tokens, usage.promptTokenCount),
+    outputTokens: finiteOrNull(usage.output_tokens, usage.completion_tokens, usage.candidatesTokenCount),
+    cacheReadInputTokens: finiteOrNull(usage.cache_read_input_tokens),
+    cacheWriteInputTokens: finiteOrNull(usage.cache_creation_input_tokens),
+  };
+}
+
+function buildLlmTrace(options = {}, event = {}) {
+  const usage = normalizeUsageTelemetry(event.usage);
+  return {
+    timestamp: new Date().toISOString(),
+    traceId: String(options.traceId || options.metadata?.traceId || '').trim() || null,
+    provider: event.provider || 'unknown',
+    model: event.model || options.model || null,
+    outcome: event.outcome || 'unknown',
+    latencyMs: Number.isFinite(event.latencyMs) ? Math.max(0, event.latencyMs) : null,
+    ...usage,
+    stopReason: event.stopReason || null,
+    requestId: event.requestId || null,
+    httpStatus: Number.isFinite(event.httpStatus) ? event.httpStatus : null,
+    errorCode: event.errorCode == null ? null : String(event.errorCode).slice(0, 80),
+  };
+}
+
+async function emitLlmTrace(options = {}, event = {}) {
+  if (typeof options.onTrace !== 'function') return;
+  try {
+    await options.onTrace(buildLlmTrace(options, event));
+  } catch {
+    // Observability must never break the generation path.
+  }
+}
+
+function gatewayRequestHeaders(config, env = process.env) {
+  const headers = { 'Content-Type': 'application/json' };
+  const token = env[config.credentialEnvVar];
+  if (token) headers.Authorization = `Bearer ${token}`;
+  return headers;
+}
+
+function buildGatewayMessages(options = {}) {
+  const supplied = Array.isArray(options.messages)
+    ? options.messages.filter(Boolean)
+    : [];
+  const messages = [];
+  if (options.systemPrompt) {
+    messages.push({ role: 'system', content: options.systemPrompt });
+  }
+  if (supplied.length > 0) messages.push(...supplied);
+  else messages.push({ role: 'user', content: options.userPrompt || '' });
+  return messages;
+}
+
+async function callGatewayInternal(options = {}, env = process.env) {
+  const config = getGatewayConfig(env);
+  if (!config || typeof fetch !== 'function') return null;
+  const model = resolveGatewayModel(options, config);
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), GATEWAY_TIMEOUT_MS);
+  const startedAt = Date.now();
+  try {
+    const response = await fetch(`${config.baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers: gatewayRequestHeaders(config, env),
+      body: JSON.stringify({
+        model,
+        messages: buildGatewayMessages(options),
+        max_tokens: options.maxTokens || DEFAULT_MAX_TOKENS,
+        temperature: Number.isFinite(options.temperature) ? options.temperature : 0,
+      }),
+      signal: controller.signal,
+    });
+    if (!response.ok) {
+      await emitLlmTrace(options, {
+        provider: 'gateway',
+        model,
+        outcome: 'error',
+        latencyMs: Date.now() - startedAt,
+        httpStatus: response.status,
+        errorCode: 'http_error',
+      });
+      return null;
+    }
+    const payload = await response.json();
+    const choice = payload?.choices?.[0] || {};
+    const message = choice.message || {};
+    const truncated = choice.finish_reason === 'length';
+    const raw = message.content || (truncated ? '' : message.reasoning_content) || '';
+    const text = stripCodeFences(raw);
+    if (!text) return null;
+    const result = {
+      text,
+      usage: payload?.usage || null,
+      stopReason: choice.finish_reason || null,
+      id: payload?.id || null,
+      model: payload?.model || model,
+      provider: 'gateway',
+    };
+    await emitLlmTrace(options, {
+      provider: 'gateway',
+      model: result.model,
+      outcome: 'success',
+      latencyMs: Date.now() - startedAt,
+      usage: result.usage,
+      stopReason: result.stopReason,
+      requestId: result.id,
+    });
+    return result;
+  } catch (error) {
+    const safe = buildSafeProviderError(error);
+    await emitLlmTrace(options, {
+      provider: 'gateway',
+      model,
+      outcome: 'error',
+      latencyMs: Date.now() - startedAt,
+      httpStatus: safe.status,
+      errorCode: safe.code || safe.name,
+    });
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 function getZaiApiKey(env = process.env) {
   return env.ZAI_API_KEY || env.THUMBGATE_ZAI_API_KEY || '';
 }
@@ -169,7 +346,18 @@ function getZaiModel(env = process.env) {
 
 async function callZaiInternal(options = {}, env = process.env) {
   const apiKey = getZaiApiKey(env);
-  if (!apiKey || typeof fetch !== 'function') return null;
+  const model = options.model || getZaiModel(env);
+  const startedAt = Date.now();
+  if (!apiKey || typeof fetch !== 'function') {
+    await emitLlmTrace(options, {
+      provider: 'zai',
+      model,
+      outcome: 'unavailable',
+      latencyMs: Date.now() - startedAt,
+      errorCode: !apiKey ? 'missing_api_key' : 'fetch_unavailable',
+    });
+    return null;
+  }
 
   const messages = Array.isArray(options.messages) && options.messages.length > 0
     ? options.messages
@@ -186,23 +374,52 @@ async function callZaiInternal(options = {}, env = process.env) {
         'Content-Type': 'application/json',
       },
       body: JSON.stringify({
-        model: options.model || getZaiModel(env),
+        model,
         messages,
         max_tokens: options.maxTokens || DEFAULT_MAX_TOKENS,
         temperature: Number.isFinite(options.temperature) ? options.temperature : 0,
       }),
     });
 
-    if (!response.ok) return null;
+    if (!response.ok) {
+      await emitLlmTrace(options, {
+        provider: 'zai',
+        model,
+        outcome: 'error',
+        latencyMs: Date.now() - startedAt,
+        httpStatus: response.status,
+        errorCode: 'http_error',
+      });
+      return null;
+    }
     const json = await response.json();
-    return {
+    const result = {
       text: stripCodeFences(json?.choices?.[0]?.message?.content || ''),
       usage: json?.usage || null,
       stopReason: json?.choices?.[0]?.finish_reason || null,
       id: json?.id || null,
-      model: json?.model || options.model || getZaiModel(env),
+      model: json?.model || model,
     };
-  } catch {
+    await emitLlmTrace(options, {
+      provider: 'zai',
+      model: result.model,
+      outcome: 'success',
+      latencyMs: Date.now() - startedAt,
+      usage: result.usage,
+      stopReason: result.stopReason,
+      requestId: result.id,
+    });
+    return result;
+  } catch (error) {
+    const safe = buildSafeProviderError(error);
+    await emitLlmTrace(options, {
+      provider: 'zai',
+      model,
+      outcome: 'error',
+      latencyMs: Date.now() - startedAt,
+      httpStatus: safe.status,
+      errorCode: safe.code || safe.name,
+    });
     return null;
   }
 }
@@ -211,8 +428,19 @@ async function callGeminiInternal(options = {}) {
   const env = process.env;
   const { detectInferenceBackend } = require('./local-model-profile');
   const providerMode = detectInferenceBackend(env).providerMode;
+  const provider = providerMode === 'vertex' ? 'vertex' : 'gemini';
+  const startedAt = Date.now();
 
-  if (providerMode !== 'vertex' && !env.GEMINI_API_KEY) return null;
+  if (providerMode !== 'vertex' && !env.GEMINI_API_KEY) {
+    await emitLlmTrace(options, {
+      provider,
+      model: options.model,
+      outcome: 'unavailable',
+      latencyMs: Date.now() - startedAt,
+      errorCode: 'missing_api_key',
+    });
+    return null;
+  }
 
   try {
     const { GoogleGenAI } = require('@google/genai');
@@ -251,7 +479,7 @@ async function callGeminiInternal(options = {}) {
       config,
     }));
 
-    return {
+    const result = {
       text: response.text || '',
       usage: response.usageMetadata ? {
         input_tokens: response.usageMetadata.promptTokenCount,
@@ -261,8 +489,27 @@ async function callGeminiInternal(options = {}) {
       id: null,
       model: options.model,
     };
+    await emitLlmTrace(options, {
+      provider,
+      model: result.model,
+      outcome: 'success',
+      latencyMs: Date.now() - startedAt,
+      usage: result.usage,
+      stopReason: result.stopReason,
+      requestId: result.id,
+    });
+    return result;
   } catch (err) {
-    console.error('Gemini/Vertex AI execution error:', JSON.stringify(buildSafeProviderError(err)));
+    const safe = buildSafeProviderError(err);
+    console.error('Gemini/Vertex AI execution error:', JSON.stringify(safe));
+    await emitLlmTrace(options, {
+      provider,
+      model: options.model,
+      outcome: 'error',
+      latencyMs: Date.now() - startedAt,
+      httpStatus: safe.status,
+      errorCode: safe.code || safe.name,
+    });
     return null;
   }
 }
@@ -296,7 +543,21 @@ async function callClaudeInternal(options = {}) {
   }
 
   const client = getClient();
-  if (!client) return null;
+  const model = options.model || DEFAULT_MODEL;
+  const startedAt = Date.now();
+  if (!client) {
+    if (isGatewayConfigured()) {
+      return callGatewayInternal(options);
+    }
+    await emitLlmTrace(options, {
+      provider: 'anthropic',
+      model,
+      outcome: 'unavailable',
+      latencyMs: Date.now() - startedAt,
+      errorCode: 'missing_client_or_api_key',
+    });
+    return null;
+  }
 
   try {
     const response = await runStep('llm.callClaude', {
@@ -305,14 +566,33 @@ async function callClaudeInternal(options = {}) {
     }, async () => client.messages.create(buildClaudeRequest(options)));
 
     const text = stripCodeFences(extractTextContent(response));
-    return {
+    const result = {
       text,
       usage: response?.usage || null,
       stopReason: response?.stop_reason || null,
       id: response?.id || null,
-      model: response?.model || options.model || DEFAULT_MODEL,
+      model: response?.model || model,
     };
-  } catch {
+    await emitLlmTrace(options, {
+      provider: 'anthropic',
+      model: result.model,
+      outcome: 'success',
+      latencyMs: Date.now() - startedAt,
+      usage: result.usage,
+      stopReason: result.stopReason,
+      requestId: result.id,
+    });
+    return result;
+  } catch (error) {
+    const safe = buildSafeProviderError(error);
+    await emitLlmTrace(options, {
+      provider: 'anthropic',
+      model,
+      outcome: 'error',
+      latencyMs: Date.now() - startedAt,
+      httpStatus: safe.status,
+      errorCode: safe.code || safe.name,
+    });
     return null;
   }
 }
@@ -372,6 +652,12 @@ async function callZaiJson(options = {}) {
 
 module.exports = {
   isAvailable,
+  getGatewayConfig,
+  isGatewayConfigured,
+  resolveGatewayModel,
+  describeInferenceAvailability,
+  callGatewayInternal,
+  buildGatewayMessages,
   callClaude,
   callClaudeJson,
   callZaiJson,
@@ -383,5 +669,8 @@ module.exports = {
   normalizeCacheOptions,
   buildClaudeRequest,
   buildSafeProviderError,
+  normalizeUsageTelemetry,
+  buildLlmTrace,
+  emitLlmTrace,
   MODELS,
 };

package/scripts/mcp-config.js

@@ -193,17 +193,37 @@ function publishedCliAvailable(pkgVersion) {
   return cliAvailabilityCache.get(pkgVersion);
 }
 
+/**
+ * Project-scope entries land in COMMITTED, SHARED config (.mcp.json / .cursor/mcp.json —
+ * init's own banner says the file serves every agent on the repo). A machine-absolute path
+ * there is a bug by construction: run init on machine A (or a Cowork sandbox with a home
+ * like /Users/busy-clever-newton) and the committed config breaks for every other machine,
+ * teammate, and CI runner. Observed for real on 2026-07-29.
+ *
+ * So: absolute paths may only ever go to HOME-scope config (machine-local by definition).
+ * Project scope gets a repo-relative path when the project IS the ThumbGate checkout
+ * (dogfooding unpublished source still works — project MCP servers launch with cwd at the
+ * project root), and the portable npx launcher otherwise.
+ */
+function relativeLocalMcpEntry(pkgRoot, targetDir) {
+  const rel = path.relative(targetDir, resolveLocalServerPath(pkgRoot, 'project'));
+  // Committed config must be separator-portable too.
+  return { command: 'node', args: [rel.split(path.sep).join('/')] };
+}
+
 function resolveMcpEntry({ pkgRoot, pkgVersion, scope = 'project', targetDir = pkgRoot }) {
   if (!isSourceCheckout(pkgRoot)) {
     return codexAutoUpdateMcpEntry();
   }
-  if (scope === 'home' && publishedCliAvailable(pkgVersion)) {
-    return codexAutoUpdateMcpEntry();
+  if (scope === 'home') {
+    if (publishedCliAvailable(pkgVersion)) return codexAutoUpdateMcpEntry();
+    return localMcpEntry(pkgRoot, scope);
   }
-  if (scope === 'project' && !isSameCheckoutFamily(pkgRoot, targetDir) && publishedCliAvailable(pkgVersion)) {
-    return codexAutoUpdateMcpEntry();
+  // scope === 'project': this is going into shared, committed config.
+  if (isSameCheckoutFamily(pkgRoot, targetDir)) {
+    return relativeLocalMcpEntry(pkgRoot, targetDir);
   }
-  return localMcpEntry(pkgRoot, scope);
+  return codexAutoUpdateMcpEntry();
 }
 
 module.exports = {
@@ -214,6 +234,7 @@ module.exports = {
   localMcpEntry,
   parseWorktreePaths,
   portableMcpEntry,
+  relativeLocalMcpEntry,
   resolveGitCommonDir,
   resolveLocalServerPath,
   resolveMcpEntry,