thumbgate 1.29.1 → 1.30.0

package/scripts/published-cli.js

@@ -13,6 +13,14 @@ function runtimePrefixDir(prefixDir) {
   return prefixDir || path.join(os.homedir(), '.thumbgate', 'runtime');
 }
 
+// For GENERATED SHELL COMMANDS only. Shell command strings land in shared, committed config
+// (.mcp.json entries, hook command lines), so expanding os.homedir() at generation time bakes
+// the generating machine's home into files other machines execute — /Users/alice/.thumbgate
+// fails with a permission error on bob's machine. shellQuote uses double quotes, so a literal
+// $HOME expands at RUNTIME on whichever machine runs the command. Non-shell consumers
+// (execFileSync paths) must keep using runtimePrefixDir, which returns a real filesystem path.
+const SHELL_RUNTIME_PREFIX = '$HOME/.thumbgate/runtime';
+
 function installedRuntimeBin(prefixDir) {
   return path.join(runtimePrefixDir(prefixDir), 'node_modules', '.bin', 'thumbgate');
 }
@@ -32,7 +40,9 @@ function publishedCliArgs(pkgVersion, commandArgs = [], options = {}) {
 }
 
 function publishedCliShellCommand(pkgVersion, commandArgs = [], options = {}) {
-  const prefixDir = runtimePrefixDir(options.prefixDir);
+  // Default to the runtime-expanded $HOME form; an explicit options.prefixDir (tests,
+  // throwaway prefixes) is honoured verbatim.
+  const prefixDir = options.prefixDir || SHELL_RUNTIME_PREFIX;
   const runtimeBin = installedRuntimeBin(prefixDir);
   const escapedArgs = commandArgs.map(shellQuote).join(' ');
   const fastPath = `[ -x ${shellQuote(runtimeBin)} ] && exec ${shellQuote(runtimeBin)}${escapedArgs ? ` ${escapedArgs}` : ''}`;

package/scripts/refresh-proof-pack.js

@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * refresh-proof-pack.js — regenerate (or check) the public evaluation scorecard.
+ *
+ * Cadence (repo policy): GitHub Actions schedule is limited to CodeQL. Noncritical
+ * loops run via workflow_dispatch or local LaunchAgent:
+ *   npm run proof-pack:refresh          # write public/eval-scorecard.html
+ *   npm run proof-pack:refresh:check    # CI gate: metrics must still match
+ *   npm run proof-pack:schedule         # install daily local LaunchAgent
+ *
+ * Isolation: generation goes through generate-eval-scorecard → thumbgate-bench
+ * isolated runtime (strict enforcement pinned).
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const PROJECT_ROOT = path.resolve(__dirname, '..');
+const SCORECARD_HTML = path.join(PROJECT_ROOT, 'public', 'eval-scorecard.html');
+const SCORECARD_JSON = path.join(PROJECT_ROOT, 'public', 'eval-scorecard.json');
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    write: false,
+    check: false,
+    json: false,
+    help: false,
+    minScore: 90,
+  };
+  for (const arg of argv) {
+    if (arg === '--write') args.write = true;
+    else if (arg === '--check') args.check = true;
+    else if (arg === '--json') args.json = true;
+    else if (arg === '--help' || arg === '-h') args.help = true;
+    else if (arg.startsWith('--min-score=')) {
+      args.minScore = Number(arg.slice('--min-score='.length));
+    }
+  }
+  if (!args.write && !args.check) {
+    // Default to write for operator cadence runs.
+    args.write = true;
+  }
+  return args;
+}
+
+function printHelp() {
+  console.log(`Usage: node scripts/refresh-proof-pack.js [--write] [--check] [--json] [--min-score=90]
+
+  --write   Regenerate public/eval-scorecard.html (+ .json sidecar)
+  --check   Fail if committed scorecard metrics diverge from a fresh bench run
+  --json    Print machine-readable summary to stdout
+  --min-score=N  Minimum composite score (default 90)
+`);
+}
+
+function extractMetricsFromHtml(html) {
+  const metrics = {};
+  // Prefer JSON-LD Dataset variableMeasured
+  const ldMatch = html.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/);
+  if (ldMatch) {
+    try {
+      const ld = JSON.parse(ldMatch[1]);
+      const vars = Array.isArray(ld.variableMeasured) ? ld.variableMeasured : [];
+      for (const item of vars) {
+        if (item && item.name != null) metrics[item.name] = item.value;
+      }
+    } catch {
+      // fall through to regex
+    }
+  }
+  const scoreMatch = html.match(/composite score <strong>([^<]+)<\/strong>/i)
+    || html.match(/composite score[^0-9]*([0-9]+)/i);
+  if (scoreMatch && metrics.score == null) {
+    metrics.score = Number(scoreMatch[1]);
+  }
+  const passMatch = html.match(/Overall:\s*<span class="(good|bad)">(PASSED|FAILED)<\/span>/i);
+  if (passMatch) {
+    metrics.passedLabel = passMatch[2].toUpperCase();
+  }
+  return metrics;
+}
+
+function normalizeMetrics(metrics = {}) {
+  const keys = [
+    'score',
+    'taskSuccessRate',
+    'unsafeActionRate',
+    'blockedUnsafeRate',
+    'capabilityRate',
+    'falseBlockRate',
+    'replayStability',
+  ];
+  const out = {};
+  for (const key of keys) {
+    if (metrics[key] == null || metrics[key] === '') continue;
+    const n = Number(metrics[key]);
+    out[key] = Number.isFinite(n) ? Number(n.toFixed(4)) : metrics[key];
+  }
+  return out;
+}
+
+function metricsEqual(a, b, options = {}) {
+  const left = normalizeMetrics(a);
+  const right = normalizeMetrics(b);
+  // When comparing a committed HTML extract to a fresh bench report, only
+  // assert keys present on the committed side (JSON-LD may omit some rates).
+  const keys = options.keys
+    || (options.committedOnly
+      ? Object.keys(left)
+      : [...new Set([...Object.keys(left), ...Object.keys(right)])]);
+  const diffs = [];
+  for (const key of keys) {
+    if (left[key] !== right[key]) {
+      diffs.push({ key, committed: left[key], fresh: right[key] });
+    }
+  }
+  return { equal: diffs.length === 0, diffs, left, right };
+}
+
+function runFreshBench() {
+  const { generate, runBench } = require('./generate-eval-scorecard');
+  // Prefer direct bench for metrics; generate for write path.
+  let report;
+  try {
+    report = runBench();
+  } catch {
+    // generate also runs the bench
+    report = null;
+  }
+  return { generate, report };
+}
+
+function buildSidecar(report, version, nowIso) {
+  const metrics = report.metrics || report;
+  return {
+    generatedAt: nowIso,
+    version,
+    sourcePath: report.sourcePath || 'bench/thumbgate-bench.json',
+    passed: report.passed !== false,
+    isolatedRuntime: report.isolatedRuntime !== false,
+    metrics: normalizeMetrics(metrics),
+    scenarioCount: Array.isArray(report.scenarios) ? report.scenarios.length : null,
+    proofUrl: 'https://thumbgate.ai/eval-scorecard',
+  };
+}
+
+function refreshWrite(options = {}) {
+  const { generate } = require('./generate-eval-scorecard');
+  const now = options.now instanceof Date ? options.now : new Date();
+  const result = generate({
+    now,
+    outputPath: options.outputPath || SCORECARD_HTML,
+  });
+  const version = options.version || require(path.join(PROJECT_ROOT, 'package.json')).version;
+  const sidecar = buildSidecar(result.report, version, now.toISOString());
+  const sidecarPath = options.sidecarPath || SCORECARD_JSON;
+  fs.writeFileSync(sidecarPath, `${JSON.stringify(sidecar, null, 2)}\n`, 'utf8');
+  return {
+    mode: 'write',
+    htmlPath: result.outPath,
+    sidecarPath,
+    passed: result.report.passed !== false,
+    metrics: sidecar.metrics,
+    score: sidecar.metrics.score,
+  };
+}
+
+function refreshCheck(options = {}) {
+  const htmlPath = options.htmlPath || SCORECARD_HTML;
+  if (!fs.existsSync(htmlPath)) {
+    throw new Error(`Missing committed scorecard: ${htmlPath}`);
+  }
+  const committedHtml = fs.readFileSync(htmlPath, 'utf8');
+  const committed = normalizeMetrics(extractMetricsFromHtml(committedHtml));
+
+  const { runBench } = require('./generate-eval-scorecard');
+  const report = options.report || runBench();
+  const fresh = normalizeMetrics(report.metrics || report);
+  const comparison = metricsEqual(committed, fresh, { committedOnly: true });
+  const score = Number(fresh.score ?? committed.score);
+  const minScore = options.minScore ?? 90;
+  const scoreOk = Number.isFinite(score) && score >= minScore;
+  const passed = report.passed !== false && scoreOk && comparison.equal;
+
+  return {
+    mode: 'check',
+    passed,
+    scoreOk,
+    metricsMatch: comparison.equal,
+    diffs: comparison.diffs,
+    committed,
+    fresh,
+    minScore,
+    reportPassed: report.passed !== false,
+  };
+}
+
+function main(argv = process.argv.slice(2)) {
+  const args = parseArgs(argv);
+  if (args.help) {
+    printHelp();
+    return 0;
+  }
+
+  let summary;
+  if (args.check) {
+    summary = refreshCheck({ minScore: args.minScore });
+  } else {
+    summary = refreshWrite();
+    if (Number(summary.score) < args.minScore || summary.passed === false) {
+      summary.checkFailed = true;
+    }
+  }
+
+  if (args.json) {
+    console.log(JSON.stringify(summary, null, 2));
+  } else if (summary.mode === 'write') {
+    console.log(
+      `Proof pack scorecard written: ${summary.htmlPath} (score=${summary.score}, passed=${summary.passed})`,
+    );
+    console.log(`Sidecar: ${summary.sidecarPath}`);
+  } else {
+    console.log(
+      `Proof pack check: metricsMatch=${summary.metricsMatch} scoreOk=${summary.scoreOk} reportPassed=${summary.reportPassed}`,
+    );
+    if (summary.diffs.length) {
+      for (const diff of summary.diffs) {
+        console.log(`  drift ${diff.key}: committed=${diff.committed} fresh=${diff.fresh}`);
+      }
+    }
+  }
+
+  if (summary.mode === 'check' && !summary.passed) return 1;
+  if (summary.mode === 'write' && summary.checkFailed) return 1;
+  return 0;
+}
+
+if (path.resolve(process.argv[1] || '') === path.resolve(__filename)) {
+  try {
+    process.exitCode = main();
+  } catch (err) {
+    console.error(err.message || err);
+    process.exitCode = 1;
+  }
+}
+
+module.exports = {
+  parseArgs,
+  extractMetricsFromHtml,
+  normalizeMetrics,
+  metricsEqual,
+  refreshWrite,
+  refreshCheck,
+  buildSidecar,
+  main,
+  SCORECARD_HTML,
+  SCORECARD_JSON,
+};

package/scripts/risk-scorer.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 const path = require('path');
 const { resolveFeedbackDir: resolveSharedFeedbackDir } = require('./feedback-paths');
 const { requireLearnedModelsEntitlement } = require('./entitlement');
+const { stratifiedSplit, evaluate, roundReport } = require('./model-eval');
 
 const PROJECT_ROOT = path.join(__dirname, '..');
 const DEFAULT_FEEDBACK_DIR = resolveSharedFeedbackDir();
@@ -187,10 +188,16 @@ function stumpPredict(value, threshold, polarity) {
   return decision * polarity;
 }
 
-function findBestWeakLearner(examples, weights, featureNames) {
+function findBestWeakLearner(examples, weights, featureNames, options = {}) {
+  const usage = options.usage || null;
+  const maxPerFeature = Number(options.maxPerFeature || Infinity);
   let best = null;
 
   featureNames.forEach((feature) => {
+    // Diversity bound: once a feature has been split on maxPerFeature times, later rounds must
+    // find signal elsewhere or stop. Off by default (Infinity) so existing behaviour is bit
+    // identical unless a caller opts in.
+    if (usage && Number.isFinite(maxPerFeature) && (usage.get(feature) || 0) >= maxPerFeature) return;
     const values = examples.map((example) => example.features[feature]);
     const thresholds = candidateThresholds(values);
     thresholds.forEach((threshold) => {
@@ -268,18 +275,14 @@ function buildPatternSummary(rows) {
   };
 }
 
-function trainRiskModel(rows, options = {}) {
-  requireLearnedModelsEntitlement({
-    ...(options.entitlement || {}),
-    label: 'risk-scorer AdaBoost training',
-  });
-  const registry = buildFeatureRegistry(rows, options);
-  const examples = rows.map((row) => ({
-    row,
-    label: deriveTargetRisk(row) === 1 ? 1 : -1,
-    features: extractFeatureMap(row, registry),
-  }));
-
+/**
+ * Fit the ensemble on a given set of examples.
+ *
+ * Extracted from trainRiskModel so that the held-out probe below is fitted by IDENTICAL code.
+ * If the probe were trained by a separate path, its score would describe a model we do not
+ * ship, and the resulting "generalization" number would be fiction.
+ */
+function fitBoostedModel(examples, registry, options = {}) {
   const model = {
     version: 1,
     algorithm: 'adaboost-stumps',
@@ -292,7 +295,7 @@ function trainRiskModel(rows, options = {}) {
     featureRegistry: registry,
     featureNames: examples[0] ? Object.keys(examples[0].features) : [],
     learners: [],
-    patterns: buildPatternSummary(rows),
+    patterns: options.patterns || { tags: [], domains: [], skills: [] },
     metrics: {
       trainingAccuracy: 0,
       rounds: 0,
@@ -307,9 +310,16 @@ function trainRiskModel(rows, options = {}) {
 
   let weights = normalizeWeights(Array(examples.length).fill(1));
   const rounds = Math.max(1, Math.min(12, Number(options.rounds || 8)));
+  // Boosting is free to pick the same feature every round. On the real corpus it did exactly
+  // that — six of eight stumps split on `recentTrend`, so an "ensemble" was in practice a
+  // one-feature model of how the session had been going lately. maxPerFeature bounds that.
+  // Default is Infinity (historical behaviour); enabling it is an evidence-based decision made
+  // by comparing held-out lift, not an assumption. See docs/ML-EVALUATION.md.
+  const maxPerFeature = Number(options.maxPerFeature || Infinity);
+  const usage = new Map();
 
   for (let round = 0; round < rounds; round += 1) {
-    const learner = findBestWeakLearner(examples, weights, model.featureNames);
+    const learner = findBestWeakLearner(examples, weights, model.featureNames, { usage, maxPerFeature });
     if (!learner) break;
 
     const clippedError = Math.min(Math.max(learner.error, 1e-6), 1 - 1e-6);
@@ -322,6 +332,7 @@ function trainRiskModel(rows, options = {}) {
       polarity: learner.polarity,
       alpha: Math.round(alpha * 1000) / 1000,
     });
+    usage.set(learner.feature, (usage.get(learner.feature) || 0) + 1);
 
     weights = normalizeWeights(weights.map((weight, index) => (
       weight * Math.exp(-alpha * examples[index].label * learner.predictions[index])
@@ -334,6 +345,119 @@ function trainRiskModel(rows, options = {}) {
   return model;
 }
 
+/** Score a fitted model over examples as (probability, label) pairs for model-eval. */
+function scorePairs(model, examples) {
+  return examples.map((example) => ({
+    probability: predictRisk(model, example.row).probability,
+    label: example.label === 1 ? 1 : 0,
+  }));
+}
+
+/**
+ * Held-out estimate of what this training procedure generalizes to.
+ *
+ * Split on CONTENT, not position or an RNG. Content hashing gives two properties we need:
+ * the split is reproducible on every machine, and near-duplicate rows (which this corpus has
+ * plenty of, since similar actions recur) land in the SAME fold instead of leaking an answer
+ * from train into test and inflating the score.
+ */
+function holdoutEvaluation(examples, registry, options = {}) {
+  const { train, test } = stratifiedSplit(examples, {
+    testFraction: Number(options.testFraction || 0.25),
+    // splitSalt exists so the SAME corpus can be re-split many ways for repeated-resampling
+    // validation. A single split of a few hundred rows cannot distinguish a real improvement
+    // from sampling noise, and picking a configuration on one split is how you overfit the
+    // validation set itself.
+    // Key on the extracted feature vector ALONE. The model observes nothing else, so two rows
+    // with identical features are the same input to it and must share a fold. An earlier
+    // version also mixed in the raw `context` string, which split rows whose different prose
+    // maps to identical features — recreating the very leakage this splitter exists to stop.
+    keyFn: options.groupKeyFn
+      ? (example) => JSON.stringify([options.splitSalt || '', options.groupKeyFn(example)])
+      : (example) => JSON.stringify([options.splitSalt || '', example.features]),
+  });
+
+  // Saying "not measurable" is the honest output for a corpus too small or too one-sided to
+  // hold anything out. Reporting a number here would be worse than reporting nothing.
+  if (test.length === 0 || train.length < 6) {
+    return { available: false, reason: 'corpus-too-small-or-single-class' };
+  }
+
+  // REBUILD THE VOCABULARY FROM THE TRAINING FOLD ONLY.
+  //
+  // buildFeatureRegistry picks top tags and skills by frequency. Deriving it from the whole
+  // corpus lets held-out rows decide which features exist — a transductive fit. The probe would
+  // see vocabulary chosen with knowledge of the test fold, and the "held-out" number would then
+  // describe a procedure we never run in production. Registry and features come from train only.
+  const foldRegistry = buildFeatureRegistry(train.map((example) => example.row), options);
+  const trainRefit = train.map((example) => ({
+    row: example.row,
+    label: example.label,
+    features: extractFeatureMap(example.row, foldRegistry),
+  }));
+
+  const probe = fitBoostedModel(trainRefit, foldRegistry, { ...options, patterns: undefined });
+  // Test rows are scored via predictRisk, which extracts features using the probe's OWN
+  // registry — so the test fold is judged under exactly the vocabulary the probe learned.
+  const report = evaluate(scorePairs(probe, test));
+  return {
+    available: true,
+    trainCount: train.length,
+    testCount: test.length,
+    ...roundReport(report),
+  };
+}
+
+function trainRiskModel(rows, options = {}) {
+  requireLearnedModelsEntitlement({
+    ...(options.entitlement || {}),
+    label: 'risk-scorer AdaBoost training',
+  });
+  const registry = buildFeatureRegistry(rows, options);
+  const examples = rows.map((row) => ({
+    row,
+    label: deriveTargetRisk(row) === 1 ? 1 : -1,
+    features: extractFeatureMap(row, registry),
+  }));
+
+  const model = fitBoostedModel(examples, registry, {
+    ...options,
+    patterns: buildPatternSummary(rows),
+  });
+
+  // The shipped model is fitted on everything — that is the right thing to deploy. The probe
+  // above estimates what that procedure generalizes to. Both numbers are recorded, and
+  // `inSample` is labelled as such so it can never again be quoted as if it were quality.
+  model.metrics.inSample = roundReport(evaluate(scorePairs(model, examples)));
+  if (options.skipHoldout !== true) {
+    // TWO held-out estimates, because they answer different questions and only reporting the
+    // friendlier one would repeat the original sin of this file.
+    //
+    //   holdout             — IID split on the full feature vector. "Does this work on new
+    //                         rows of the kinds we have seen?"  Measured 2026-07-28: +0.091
+    //                         lift, AUC 0.884, 12/12 resamples beat baseline.
+    //
+    //   holdoutNovelContext — split by coarse content group, so whole action categories are
+    //                         absent from training. "Does this work on kinds of actions we
+    //                         have never seen?"  Measured: NEGATIVE lift.
+    //
+    // For a firewall the second question is the one that matters most — novel attacks are by
+    // definition unfamiliar — so the pessimistic number is recorded next to the optimistic one
+    // permanently, and docs/ML-EVALUATION.md explains the gap.
+    model.metrics.holdout = holdoutEvaluation(examples, registry, options);
+    model.metrics.holdoutNovelContext = holdoutEvaluation(examples, registry, {
+      ...options,
+      groupKeyFn: (example) => JSON.stringify([
+        example.row && example.row.context,
+        example.row && example.row.domain,
+        example.row && example.row.skill,
+        example.row && example.row.targetTags,
+      ]),
+    });
+  }
+  return model;
+}
+
 function rawScore(model, row) {
   if (!model || !model.featureRegistry) {
     return 0;
@@ -457,6 +581,11 @@ module.exports = {
   trainAndPersistRiskModel,
   trainRiskModel,
   getRiskSummary,
+  // Exported for the evaluation harness (scripts/eval-risk-model.js) and its tests: measuring
+  // this model requires fitting it on a fold, which requires reaching the fit step directly.
+  fitBoostedModel,
+  holdoutEvaluation,
+  scorePairs,
 };
 
 if (require.main === module) {