thumbgate 1.28.4 → 1.29.2

package/scripts/meta-agent-loop.js

@@ -581,6 +581,20 @@ async function main() {
   const args = process.argv.slice(2);
   const dryRun = args.includes('--dry-run');
   const verbose = args.includes('--verbose') || args.includes('-v');
+  let hookMode = args.includes('--hook');
+
+  // Settings may already be cached by a running agent process. Detect the
+  // current Stop payload as well, so the legacy command (without --hook) is
+  // immediately JSON-safe before the host reloads its settings.
+  if (!hookMode && !process.stdin.isTTY) {
+    try {
+      const raw = fs.readFileSync(0, 'utf8');
+      const payload = raw ? JSON.parse(raw) : {};
+      hookMode = payload.hook_event_name === 'Stop';
+    } catch {
+      // Normal CLI invocation with empty/non-JSON stdin.
+    }
+  }
 
   if (args.includes('--status')) {
     const status = getMetaAgentStatus();
@@ -593,9 +607,13 @@ async function main() {
   }
 
   const mode = dryRun ? 'DRY RUN' : 'LIVE';
-  console.log(`Meta-agent loop starting [${mode}]...`);
+  if (!hookMode) console.log(`Meta-agent loop starting [${mode}]...`);
+
+  const manifest = await runMetaAgentLoop({ dryRun, verbose: hookMode ? false : true });
 
-  const manifest = await runMetaAgentLoop({ dryRun, verbose: verbose || true });
+  // Stop hooks may emit only empty stdout or a valid JSON object. The loop's
+  // work is intentionally silent in hook mode.
+  if (hookMode) return;
 
   console.log(`Run ID        : ${manifest.runId}`);
   console.log(`Analysis mode : ${manifest.analysisMode}`);

package/scripts/observability-env.js

@@ -0,0 +1,139 @@
+'use strict';
+
+/**
+ * Load operator-local observability credentials into process.env.
+ *
+ * Sources (first wins per key, never overwrite an already-set env var):
+ *  1. process.env
+ *  2. ~/.config/thumbgate/observability.json
+ *  3. ~/.config/thumbgate/operator.json (operatorKey / baseUrl only)
+ *  4. Stripe managed secret files via resolveStripeSecretKey
+ *
+ * The JSON file is gitignored operator state. Never print secret values.
+ */
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+
+const OBSERVABILITY_CONFIG_PATH = path.join(os.homedir(), '.config', 'thumbgate', 'observability.json');
+const OPERATOR_CONFIG_PATH = path.join(os.homedir(), '.config', 'thumbgate', 'operator.json');
+
+const JSON_KEY_TO_ENV = Object.freeze({
+  stripeSecretKey: 'STRIPE_SECRET_KEY',
+  plausibleApiKey: 'PLAUSIBLE_API_KEY',
+  plausibleSiteId: 'PLAUSIBLE_SITE_ID',
+  plausibleSiteIds: 'PLAUSIBLE_SITE_IDS',
+  plausibleRegisteredDomains: 'THUMBGATE_PLAUSIBLE_REGISTERED_DOMAINS',
+  posthogPersonalApiKey: 'POSTHOG_PERSONAL_API_KEY',
+  posthogProjectId: 'POSTHOG_PROJECT_ID',
+  operatorKey: 'THUMBGATE_OPERATOR_KEY',
+  apiKey: 'THUMBGATE_API_KEY',
+  publicAppOrigin: 'THUMBGATE_PUBLIC_APP_ORIGIN',
+  billingApiBaseUrl: 'THUMBGATE_BILLING_API_BASE_URL',
+});
+
+function normalizeText(value) {
+  if (value === undefined || value === null) return null;
+  const text = String(value).trim();
+  return text || null;
+}
+
+function readJsonFile(filePath) {
+  try {
+    if (!filePath || !fs.existsSync(filePath)) return null;
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function applyJsonToEnv(json, env) {
+  if (!json || typeof json !== 'object') return [];
+  const applied = [];
+  for (const [jsonKey, envKey] of Object.entries(JSON_KEY_TO_ENV)) {
+    if (normalizeText(env[envKey])) continue;
+    const value = normalizeText(json[jsonKey]);
+    if (!value) continue;
+    env[envKey] = value;
+    applied.push(envKey);
+  }
+  return applied;
+}
+
+function loadObservabilityEnv({
+  env = process.env,
+  observabilityPath = OBSERVABILITY_CONFIG_PATH,
+  operatorPath = OPERATOR_CONFIG_PATH,
+  applyStripeManagedFiles = true,
+} = {}) {
+  const applied = [];
+
+  const observability = readJsonFile(observabilityPath);
+  applied.push(...applyJsonToEnv(observability, env));
+
+  const operator = readJsonFile(operatorPath);
+  if (operator) {
+    if (!normalizeText(env.THUMBGATE_OPERATOR_KEY) && normalizeText(operator.operatorKey)) {
+      env.THUMBGATE_OPERATOR_KEY = String(operator.operatorKey).trim();
+      applied.push('THUMBGATE_OPERATOR_KEY');
+    }
+    if (!normalizeText(env.THUMBGATE_BILLING_API_BASE_URL) && normalizeText(operator.baseUrl)) {
+      env.THUMBGATE_BILLING_API_BASE_URL = String(operator.baseUrl).trim();
+      applied.push('THUMBGATE_BILLING_API_BASE_URL');
+    }
+  }
+
+  if (applyStripeManagedFiles && !normalizeText(env.STRIPE_SECRET_KEY)) {
+    try {
+      const { resolveStripeSecretKey } = require('./stripe-credentials');
+      const resolved = resolveStripeSecretKey({ env });
+      if (resolved.secretKey) {
+        env.STRIPE_SECRET_KEY = resolved.secretKey;
+        applied.push('STRIPE_SECRET_KEY');
+      }
+    } catch {
+      // stripe-credentials optional at load time
+    }
+  }
+
+  // Product-primary Plausible domain is always part of the registered set for
+  // doctor/automation unless the operator explicitly overrides site id.
+  if (!normalizeText(env.PLAUSIBLE_SITE_ID) && !normalizeText(env.THUMBGATE_PLAUSIBLE_REGISTERED_DOMAINS)) {
+    env.THUMBGATE_PLAUSIBLE_REGISTERED_DOMAINS = 'thumbgate.ai,thumbgate-production.up.railway.app';
+    applied.push('THUMBGATE_PLAUSIBLE_REGISTERED_DOMAINS');
+  }
+
+  return {
+    applied: [...new Set(applied)],
+    observabilityPath,
+    operatorPath,
+    hasStripe: Boolean(normalizeText(env.STRIPE_SECRET_KEY)),
+    hasPlausible: Boolean(normalizeText(env.PLAUSIBLE_API_KEY) && normalizeText(env.PLAUSIBLE_SITE_ID)),
+    hasPosthog: Boolean(normalizeText(env.POSTHOG_PERSONAL_API_KEY) && normalizeText(env.POSTHOG_PROJECT_ID)),
+    hasOperator: Boolean(normalizeText(env.THUMBGATE_OPERATOR_KEY) || normalizeText(env.THUMBGATE_API_KEY)),
+  };
+}
+
+function observabilityConfigTemplate() {
+  return {
+    stripeSecretKey: '',
+    plausibleApiKey: '',
+    plausibleSiteId: 'thumbgate.ai',
+    plausibleRegisteredDomains: 'thumbgate.ai,thumbgate-production.up.railway.app',
+    posthogPersonalApiKey: '',
+    posthogProjectId: '',
+    publicAppOrigin: 'https://thumbgate.ai',
+    billingApiBaseUrl: 'https://thumbgate-production.up.railway.app',
+  };
+}
+
+module.exports = {
+  OBSERVABILITY_CONFIG_PATH,
+  OPERATOR_CONFIG_PATH,
+  JSON_KEY_TO_ENV,
+  loadObservabilityEnv,
+  observabilityConfigTemplate,
+  readJsonFile,
+  normalizeText,
+};

package/scripts/observability-setup.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const os = require('node:os');
+const {
+  OBSERVABILITY_CONFIG_PATH,
+  observabilityConfigTemplate,
+  loadObservabilityEnv,
+} = require('./observability-env');
+
+function ensureDir(filePath) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+}
+
+function main(argv = process.argv.slice(2)) {
+  const write = argv.includes('--write');
+  const print = argv.includes('--print') || !write;
+  const template = observabilityConfigTemplate();
+
+  if (print && !write) {
+    process.stdout.write(`${JSON.stringify(template, null, 2)}\n`);
+    process.stdout.write(
+      `\n# Write to ${OBSERVABILITY_CONFIG_PATH} with --write after filling secrets.\n` +
+      '# Never commit this file. Doctor/revenue tools load it automatically.\n'
+    );
+  }
+
+  if (write) {
+    ensureDir(OBSERVABILITY_CONFIG_PATH);
+    if (fs.existsSync(OBSERVABILITY_CONFIG_PATH)) {
+      process.stderr.write(`Refusing to overwrite existing ${OBSERVABILITY_CONFIG_PATH}\n`);
+      process.exitCode = 2;
+      return;
+    }
+    fs.writeFileSync(OBSERVABILITY_CONFIG_PATH, `${JSON.stringify(template, null, 2)}\n`, { mode: 0o600 });
+    process.stdout.write(`Wrote template ${OBSERVABILITY_CONFIG_PATH}\n`);
+  }
+
+  const status = loadObservabilityEnv({ env: { ...process.env } });
+  process.stdout.write(JSON.stringify({
+    configPath: OBSERVABILITY_CONFIG_PATH,
+    hasStripe: status.hasStripe,
+    hasPlausible: status.hasPlausible,
+    hasPosthog: status.hasPosthog,
+    hasOperator: status.hasOperator,
+  }, null, 2) + '\n');
+}
+
+if (require('node:path').resolve(process.argv[1] || '') === require('node:path').resolve(__filename)) {
+  main();
+}
+
+module.exports = { main };

package/scripts/plausible-domain-config.js

@@ -30,7 +30,11 @@ function getConfiguredRegisteredDomains(env = process.env) {
     ...splitDomains(env.PLAUSIBLE_REGISTERED_DOMAINS),
   ].map(normalizeDomain).filter(Boolean);
 
+  // Both product surfaces are first-class Plausible site ids for ThumbGate.
+  // Emitting data-domain=thumbgate.ai while only registering the Railway host
+  // previously made primary-domain traffic invisible to automation.
   return [...new Set([
+    PRIMARY_PLAUSIBLE_DOMAIN,
     FALLBACK_REGISTERED_PLAUSIBLE_DOMAIN,
     ...configured,
   ])];

package/scripts/prompt-eval.js

@@ -143,6 +143,8 @@ function handleRejectExpectation(checks, result, expected) {
 
   const wasRejected = result.accepted === false
     || result.status === 'rejected'
+    || result.status === 'clarification_required'
+    || result.needsClarification === true
     || result.actionType === 'no-action';
   checks.push({
     criterion: 'shouldReject',
@@ -466,8 +468,9 @@ function runSuiteObject(suite, options = {}) {
   const skipped = results.filter((r) => r.status === 'skip').length;
   const totalScore = results.length > 0
     ? Math.round(results.reduce((s, r) => s + r.score, 0) / results.length)
-    : 100;
+    : 0;
   const minScore = options.minScore ?? 80;
+  const insufficientEvidence = results.length === 0;
 
   return {
     suite: suite.name,
@@ -478,8 +481,9 @@ function runSuiteObject(suite, options = {}) {
     skipped,
     score: totalScore,
     minScore,
-    pass: totalScore >= minScore,
-    noCases: results.length === 0,
+    pass: !insufficientEvidence && totalScore >= minScore,
+    noCases: insufficientEvidence,
+    evidenceStatus: insufficientEvidence ? 'insufficient_evidence' : 'measured',
     feedbackDerived: suite.source && suite.source.type === 'feedback-log',
     generatedAt: new Date().toISOString(),
     results,
@@ -685,6 +689,7 @@ function runSuite(suitePath = DEFAULT_SUITE, options = {}) {
 
 function compareReports(currentReport, baselineReport) {
   const baselineById = new Map((baselineReport?.results || []).map((result) => [result.id, result]));
+  const currentById = new Map((currentReport?.results || []).map((result) => [result.id, result]));
   const regressions = [];
   const improvements = [];
 
@@ -717,12 +722,83 @@ function compareReports(currentReport, baselineReport) {
     }
   }
 
+  for (const baseline of baselineReport?.results || []) {
+    if (currentById.has(baseline.id)) continue;
+    regressions.push({
+      id: baseline.id,
+      baselineScore: baseline.score,
+      currentScore: null,
+      delta: null,
+      baselineStatus: baseline.status,
+      currentStatus: 'missing',
+    });
+  }
+
   return {
     baselineSuite: baselineReport?.suite || null,
     baselineScore: Number.isFinite(Number(baselineReport?.score)) ? Number(baselineReport.score) : null,
     scoreDelta: Number.isFinite(Number(baselineReport?.score)) ? currentReport.score - Number(baselineReport.score) : null,
     regressions,
     improvements,
+    baselineCases: baselineById.size,
+    currentCases: currentById.size,
+    baselineCoverageRate: baselineById.size
+      ? Math.round((Array.from(baselineById.keys()).filter((id) => currentById.has(id)).length / baselineById.size) * 10000) / 10000
+      : null,
+  };
+}
+
+function logSafeResult(result, index) {
+  const allowedStatuses = new Set(['pass', 'fail', 'error', 'skip']);
+  return {
+    case: index + 1,
+    status: allowedStatuses.has(result.status) ? result.status : 'error',
+    score: Number(result.score || 0),
+    passCount: Number(result.passCount || 0),
+    totalChecks: Number(result.totalChecks || 0),
+  };
+}
+
+function logSafeReport(report, suite, fromFeedback) {
+  const source = suite?.source || {};
+  const comparison = report.comparison
+    ? {
+        baselineScore: Number(report.comparison.baselineScore || 0),
+        scoreDelta: Number(report.comparison.scoreDelta || 0),
+        regressionCount: Number(report.comparison.regressions?.length || 0),
+        improvementCount: Number(report.comparison.improvements?.length || 0),
+        baselineCases: Number(report.comparison.baselineCases || 0),
+        currentCases: Number(report.comparison.currentCases || 0),
+        baselineCoverageRate: report.comparison.baselineCoverageRate,
+      }
+    : undefined;
+  return {
+    suiteType: fromFeedback ? 'feedback-derived' : 'configured',
+    total: Number(report.total || 0),
+    passed: Number(report.passed || 0),
+    failed: Number(report.failed || 0),
+    errors: Number(report.errors || 0),
+    skipped: Number(report.skipped || 0),
+    score: Number(report.score || 0),
+    minScore: Number(report.minScore || 0),
+    pass: report.pass === true,
+    noCases: report.noCases === true,
+    evidenceStatus: report.evidenceStatus === 'measured' ? 'measured' : 'insufficient_evidence',
+    feedbackDerived: report.feedbackDerived === true,
+    syntheticCount: Number(report.syntheticCount || 0),
+    comparison,
+    results: (report.results || []).map(logSafeResult),
+    suiteDefinition: fromFeedback
+      ? {
+          version: Number(suite?.version || 0),
+          source: {
+            type: 'feedback-log',
+            totalEntries: Number(source.totalEntries || 0),
+            selectedCases: Number(source.selectedCases || 0),
+          },
+          evaluationCount: Number(suite?.evaluations?.length || 0),
+        }
+      : undefined,
   };
 }
 
@@ -847,7 +923,7 @@ if (isCliInvocation()) {
   }
 
   if (json) {
-    console.log(JSON.stringify({ ...report, suiteDefinition: fromFeedback ? suite : undefined }, null, 2));
+    console.log(JSON.stringify(logSafeReport(report, suite, fromFeedback), null, 2));
   } else {
     console.log(`\n${report.suite}`);
     console.log('='.repeat(50));
@@ -883,6 +959,7 @@ module.exports = {
   gradeOutput,
   loadSuite,
   loadReport,
+  logSafeReport,
   compareReports,
   readJsonl,
   runEvaluation,