thumbgate 1.29.2 → 1.30.0

package/scripts/risk-scorer.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 const path = require('path');
 const { resolveFeedbackDir: resolveSharedFeedbackDir } = require('./feedback-paths');
 const { requireLearnedModelsEntitlement } = require('./entitlement');
+const { stratifiedSplit, evaluate, roundReport } = require('./model-eval');
 
 const PROJECT_ROOT = path.join(__dirname, '..');
 const DEFAULT_FEEDBACK_DIR = resolveSharedFeedbackDir();
@@ -187,10 +188,16 @@ function stumpPredict(value, threshold, polarity) {
   return decision * polarity;
 }
 
-function findBestWeakLearner(examples, weights, featureNames) {
+function findBestWeakLearner(examples, weights, featureNames, options = {}) {
+  const usage = options.usage || null;
+  const maxPerFeature = Number(options.maxPerFeature || Infinity);
   let best = null;
 
   featureNames.forEach((feature) => {
+    // Diversity bound: once a feature has been split on maxPerFeature times, later rounds must
+    // find signal elsewhere or stop. Off by default (Infinity) so existing behaviour is bit
+    // identical unless a caller opts in.
+    if (usage && Number.isFinite(maxPerFeature) && (usage.get(feature) || 0) >= maxPerFeature) return;
     const values = examples.map((example) => example.features[feature]);
     const thresholds = candidateThresholds(values);
     thresholds.forEach((threshold) => {
@@ -268,18 +275,14 @@ function buildPatternSummary(rows) {
   };
 }
 
-function trainRiskModel(rows, options = {}) {
-  requireLearnedModelsEntitlement({
-    ...(options.entitlement || {}),
-    label: 'risk-scorer AdaBoost training',
-  });
-  const registry = buildFeatureRegistry(rows, options);
-  const examples = rows.map((row) => ({
-    row,
-    label: deriveTargetRisk(row) === 1 ? 1 : -1,
-    features: extractFeatureMap(row, registry),
-  }));
-
+/**
+ * Fit the ensemble on a given set of examples.
+ *
+ * Extracted from trainRiskModel so that the held-out probe below is fitted by IDENTICAL code.
+ * If the probe were trained by a separate path, its score would describe a model we do not
+ * ship, and the resulting "generalization" number would be fiction.
+ */
+function fitBoostedModel(examples, registry, options = {}) {
   const model = {
     version: 1,
     algorithm: 'adaboost-stumps',
@@ -292,7 +295,7 @@ function trainRiskModel(rows, options = {}) {
     featureRegistry: registry,
     featureNames: examples[0] ? Object.keys(examples[0].features) : [],
     learners: [],
-    patterns: buildPatternSummary(rows),
+    patterns: options.patterns || { tags: [], domains: [], skills: [] },
     metrics: {
       trainingAccuracy: 0,
       rounds: 0,
@@ -307,9 +310,16 @@ function trainRiskModel(rows, options = {}) {
 
   let weights = normalizeWeights(Array(examples.length).fill(1));
   const rounds = Math.max(1, Math.min(12, Number(options.rounds || 8)));
+  // Boosting is free to pick the same feature every round. On the real corpus it did exactly
+  // that — six of eight stumps split on `recentTrend`, so an "ensemble" was in practice a
+  // one-feature model of how the session had been going lately. maxPerFeature bounds that.
+  // Default is Infinity (historical behaviour); enabling it is an evidence-based decision made
+  // by comparing held-out lift, not an assumption. See docs/ML-EVALUATION.md.
+  const maxPerFeature = Number(options.maxPerFeature || Infinity);
+  const usage = new Map();
 
   for (let round = 0; round < rounds; round += 1) {
-    const learner = findBestWeakLearner(examples, weights, model.featureNames);
+    const learner = findBestWeakLearner(examples, weights, model.featureNames, { usage, maxPerFeature });
     if (!learner) break;
 
     const clippedError = Math.min(Math.max(learner.error, 1e-6), 1 - 1e-6);
@@ -322,6 +332,7 @@ function trainRiskModel(rows, options = {}) {
       polarity: learner.polarity,
       alpha: Math.round(alpha * 1000) / 1000,
     });
+    usage.set(learner.feature, (usage.get(learner.feature) || 0) + 1);
 
     weights = normalizeWeights(weights.map((weight, index) => (
       weight * Math.exp(-alpha * examples[index].label * learner.predictions[index])
@@ -334,6 +345,119 @@ function trainRiskModel(rows, options = {}) {
   return model;
 }
 
+/** Score a fitted model over examples as (probability, label) pairs for model-eval. */
+function scorePairs(model, examples) {
+  return examples.map((example) => ({
+    probability: predictRisk(model, example.row).probability,
+    label: example.label === 1 ? 1 : 0,
+  }));
+}
+
+/**
+ * Held-out estimate of what this training procedure generalizes to.
+ *
+ * Split on CONTENT, not position or an RNG. Content hashing gives two properties we need:
+ * the split is reproducible on every machine, and near-duplicate rows (which this corpus has
+ * plenty of, since similar actions recur) land in the SAME fold instead of leaking an answer
+ * from train into test and inflating the score.
+ */
+function holdoutEvaluation(examples, registry, options = {}) {
+  const { train, test } = stratifiedSplit(examples, {
+    testFraction: Number(options.testFraction || 0.25),
+    // splitSalt exists so the SAME corpus can be re-split many ways for repeated-resampling
+    // validation. A single split of a few hundred rows cannot distinguish a real improvement
+    // from sampling noise, and picking a configuration on one split is how you overfit the
+    // validation set itself.
+    // Key on the extracted feature vector ALONE. The model observes nothing else, so two rows
+    // with identical features are the same input to it and must share a fold. An earlier
+    // version also mixed in the raw `context` string, which split rows whose different prose
+    // maps to identical features — recreating the very leakage this splitter exists to stop.
+    keyFn: options.groupKeyFn
+      ? (example) => JSON.stringify([options.splitSalt || '', options.groupKeyFn(example)])
+      : (example) => JSON.stringify([options.splitSalt || '', example.features]),
+  });
+
+  // Saying "not measurable" is the honest output for a corpus too small or too one-sided to
+  // hold anything out. Reporting a number here would be worse than reporting nothing.
+  if (test.length === 0 || train.length < 6) {
+    return { available: false, reason: 'corpus-too-small-or-single-class' };
+  }
+
+  // REBUILD THE VOCABULARY FROM THE TRAINING FOLD ONLY.
+  //
+  // buildFeatureRegistry picks top tags and skills by frequency. Deriving it from the whole
+  // corpus lets held-out rows decide which features exist — a transductive fit. The probe would
+  // see vocabulary chosen with knowledge of the test fold, and the "held-out" number would then
+  // describe a procedure we never run in production. Registry and features come from train only.
+  const foldRegistry = buildFeatureRegistry(train.map((example) => example.row), options);
+  const trainRefit = train.map((example) => ({
+    row: example.row,
+    label: example.label,
+    features: extractFeatureMap(example.row, foldRegistry),
+  }));
+
+  const probe = fitBoostedModel(trainRefit, foldRegistry, { ...options, patterns: undefined });
+  // Test rows are scored via predictRisk, which extracts features using the probe's OWN
+  // registry — so the test fold is judged under exactly the vocabulary the probe learned.
+  const report = evaluate(scorePairs(probe, test));
+  return {
+    available: true,
+    trainCount: train.length,
+    testCount: test.length,
+    ...roundReport(report),
+  };
+}
+
+function trainRiskModel(rows, options = {}) {
+  requireLearnedModelsEntitlement({
+    ...(options.entitlement || {}),
+    label: 'risk-scorer AdaBoost training',
+  });
+  const registry = buildFeatureRegistry(rows, options);
+  const examples = rows.map((row) => ({
+    row,
+    label: deriveTargetRisk(row) === 1 ? 1 : -1,
+    features: extractFeatureMap(row, registry),
+  }));
+
+  const model = fitBoostedModel(examples, registry, {
+    ...options,
+    patterns: buildPatternSummary(rows),
+  });
+
+  // The shipped model is fitted on everything — that is the right thing to deploy. The probe
+  // above estimates what that procedure generalizes to. Both numbers are recorded, and
+  // `inSample` is labelled as such so it can never again be quoted as if it were quality.
+  model.metrics.inSample = roundReport(evaluate(scorePairs(model, examples)));
+  if (options.skipHoldout !== true) {
+    // TWO held-out estimates, because they answer different questions and only reporting the
+    // friendlier one would repeat the original sin of this file.
+    //
+    //   holdout             — IID split on the full feature vector. "Does this work on new
+    //                         rows of the kinds we have seen?"  Measured 2026-07-28: +0.091
+    //                         lift, AUC 0.884, 12/12 resamples beat baseline.
+    //
+    //   holdoutNovelContext — split by coarse content group, so whole action categories are
+    //                         absent from training. "Does this work on kinds of actions we
+    //                         have never seen?"  Measured: NEGATIVE lift.
+    //
+    // For a firewall the second question is the one that matters most — novel attacks are by
+    // definition unfamiliar — so the pessimistic number is recorded next to the optimistic one
+    // permanently, and docs/ML-EVALUATION.md explains the gap.
+    model.metrics.holdout = holdoutEvaluation(examples, registry, options);
+    model.metrics.holdoutNovelContext = holdoutEvaluation(examples, registry, {
+      ...options,
+      groupKeyFn: (example) => JSON.stringify([
+        example.row && example.row.context,
+        example.row && example.row.domain,
+        example.row && example.row.skill,
+        example.row && example.row.targetTags,
+      ]),
+    });
+  }
+  return model;
+}
+
 function rawScore(model, row) {
   if (!model || !model.featureRegistry) {
     return 0;
@@ -457,6 +581,11 @@ module.exports = {
   trainAndPersistRiskModel,
   trainRiskModel,
   getRiskSummary,
+  // Exported for the evaluation harness (scripts/eval-risk-model.js) and its tests: measuring
+  // this model requires fitting it on a fold, which requires reaching the fit step directly.
+  fitBoostedModel,
+  holdoutEvaluation,
+  scorePairs,
 };
 
 if (require.main === module) {

package/scripts/statusline-local-stats.js

@@ -16,7 +16,7 @@ try {
   const scope = String(process.env.THUMBGATE_STATUSLINE_SCOPE || 'global').toLowerCase();
   const stats = scope !== 'project' && shouldAggregateFeedback({ env: process.env })
     ? computeAggregateFeedbackStats({ projectDir, env: process.env })
-    : analyzeFeedback();
+    : analyzeFeedback(undefined, { humanOnly: true });
   const payload = {
     ...normalizeStatsPayload(stats),
     aggregate: stats.aggregate || { enabled: false },

package/scripts/thumbgate-bench.js

@@ -268,6 +268,13 @@ function withGateRuntime(options, callback) {
     'THUMBGATE_SECRET_SCAN_PROVIDER',
     'THUMBGATE_HARNESS',
     'THUMBGATE_HARNESS_CONFIG',
+    // Isolated bench must not inherit operator license / posture / daily-cap state.
+    // Without strict enforcement, deny expectations become warn-by-default and the
+    // published scorecard is unreproducible on a clean free-tier install.
+    'THUMBGATE_STRICT_ENFORCEMENT',
+    'THUMBGATE_ENFORCE_ENTITLEMENTS',
+    'THUMBGATE_PRO_LICENSE_KEY',
+    'THUMBGATE_LICENSE_KEY',
   ]);
   const runtimeDir = options.useRuntimeState
     ? null
@@ -289,6 +296,12 @@ function withGateRuntime(options, callback) {
       process.env.THUMBGATE_ATTRIBUTED_FEEDBACK = path.join(runtimeDir, 'attributed-feedback.jsonl');
       process.env.THUMBGATE_GUARDS_PATH = path.join(runtimeDir, 'pretool-guards.json');
       process.env.THUMBGATE_SECRET_SCAN_PROVIDER = 'heuristic';
+      // Pin enforcement posture for golden deny expectations (scorecard reproducibility).
+      process.env.THUMBGATE_STRICT_ENFORCEMENT = '1';
+      // Do not hard-fail free-tier installs; isolation already avoids daily-cap state.
+      delete process.env.THUMBGATE_ENFORCE_ENTITLEMENTS;
+      delete process.env.THUMBGATE_PRO_LICENSE_KEY;
+      delete process.env.THUMBGATE_LICENSE_KEY;
       fs.mkdirSync(process.env.THUMBGATE_FEEDBACK_DIR, { recursive: true });
       fs.writeFileSync(process.env.THUMBGATE_FEEDBACK_LOG, '');
       fs.writeFileSync(process.env.THUMBGATE_ATTRIBUTED_FEEDBACK, '');

package/scripts/tool-kpi-tracker.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+'use strict';
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { resolveFeedbackDir } = require('./feedback-paths');
+const { readJsonl } = require('./fs-utils');
+
+function getKpiLogPath(options = {}) {
+  return path.join(
+    options.feedbackDir ? path.resolve(options.feedbackDir) : resolveFeedbackDir(),
+    'tool-kpi.jsonl',
+  );
+}
+
+function recordToolCall({
+  toolName,
+  serverName,
+  latencyMs,
+  success,
+  agentId,
+  metadata,
+  feedbackDir,
+} = {}) {
+  const logPath = getKpiLogPath({ feedbackDir });
+  fs.mkdirSync(path.dirname(logPath), { recursive: true });
+  const entry = {
+    id: `kpi_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`,
+    timestamp: new Date().toISOString(),
+    toolName: toolName || 'unknown',
+    serverName: serverName || 'default',
+    latencyMs: typeof latencyMs === 'number' ? latencyMs : 0,
+    success: success !== false,
+    agentId: agentId || 'unknown',
+    metadata: metadata || {},
+  };
+  fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`);
+  return entry;
+}
+
+function percentile(sorted, quantile) {
+  if (sorted.length === 0) return 0;
+  const index = Math.ceil((quantile / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, index)];
+}
+
+function computeToolKpis({ periodHours = 24, feedbackDir } = {}) {
+  const entries = readJsonl(getKpiLogPath({ feedbackDir }));
+  const cutoff = Date.now() - periodHours * 60 * 60 * 1000;
+  const recent = entries.filter((entry) => new Date(entry.timestamp).getTime() > cutoff);
+  const byTool = {};
+  for (const entry of recent) {
+    const key = entry.toolName;
+    if (!byTool[key]) {
+      byTool[key] = {
+        toolName: key,
+        calls: [],
+        successes: 0,
+        failures: 0,
+      };
+    }
+    byTool[key].calls.push(entry.latencyMs);
+    if (entry.success) byTool[key].successes += 1;
+    else byTool[key].failures += 1;
+  }
+  const tools = Object.values(byTool)
+    .map((tool) => {
+      const sorted = tool.calls.slice().sort((left, right) => left - right);
+      const total = tool.successes + tool.failures;
+      return {
+        toolName: tool.toolName,
+        requestCount: total,
+        successRate: total > 0 ? Math.round((tool.successes / total) * 1000) / 10 : 100,
+        p50: Math.round(percentile(sorted, 50)),
+        p90: Math.round(percentile(sorted, 90)),
+        p95: Math.round(percentile(sorted, 95)),
+        successes: tool.successes,
+        failures: tool.failures,
+      };
+    })
+    .sort((left, right) => right.requestCount - left.requestCount);
+
+  const byServer = {};
+  for (const entry of recent) {
+    const key = entry.serverName;
+    if (!byServer[key]) byServer[key] = { serverName: key, total: 0, successes: 0 };
+    byServer[key].total += 1;
+    if (entry.success) byServer[key].successes += 1;
+  }
+  const servers = Object.values(byServer).map((server) => ({
+    serverName: server.serverName,
+    totalCalls: server.total,
+    successRate: server.total > 0
+      ? Math.round((server.successes / server.total) * 1000) / 10
+      : 100,
+  }));
+  return {
+    periodHours,
+    totalCalls: recent.length,
+    evidenceStatus: recent.length > 0 ? 'measured' : 'insufficient_evidence',
+    tools,
+    servers,
+  };
+}
+
+function getAtRiskTools({
+  successRateThreshold = 90,
+  p95Threshold = 500,
+  periodHours = 24,
+  feedbackDir,
+} = {}) {
+  const { tools } = computeToolKpis({ periodHours, feedbackDir });
+  return tools.filter((tool) => tool.requestCount >= 3
+    && (tool.successRate < successRateThreshold || tool.p95 > p95Threshold));
+}
+
+module.exports = {
+  computeToolKpis,
+  getAtRiskTools,
+  getKpiLogPath,
+  percentile,
+  recordToolCall,
+};

package/scripts/tool-registry.js

@@ -133,8 +133,20 @@ const TASK_OUTCOME_INPUT_SCHEMA = {
   },
 };
 
+const MEMORY_SCOPE_SCHEMA = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['entityId', 'projectId', 'processId', 'sessionId'],
+  properties: {
+    entityId: { type: 'string', minLength: 1 },
+    projectId: { type: 'string', minLength: 1 },
+    processId: { type: 'string', minLength: 1 },
+    sessionId: { type: 'string', minLength: 1 },
+  },
+};
+
 const TOOLS = [
-  readOnlyTool({
+  destructiveTool({
     name: 'capture_feedback',
     description: 'Capture an up/down signal plus one line of why. Vague feedback is logged, then returned with a clarification prompt instead of memory promotion.',
     inputSchema: {
@@ -218,6 +230,9 @@ const TOOLS = [
         limit: { type: 'number', description: 'Maximum results to return (default 10)' },
         category: { type: 'string', enum: ['error', 'learning', 'preference'] },
         tags: { type: 'array', items: { type: 'string' }, description: 'Require all tags to be present on a lesson' },
+        scope: MEMORY_SCOPE_SCHEMA,
+        requireScope: { type: 'boolean', description: 'Fail closed unless a complete four-field scope is supplied.' },
+        includeShared: { type: 'boolean', description: 'Include explicitly shared memories with scoped results. Defaults true.' },
       },
     },
   }),
@@ -230,6 +245,9 @@ const TOOLS = [
         toolName: { type: 'string', description: 'The tool being called (e.g., Bash, Edit, Read)' },
         actionContext: { type: 'string', description: 'Description of what the tool call is doing' },
         maxResults: { type: 'number', description: 'Max lessons to return (default 5)' },
+        scope: MEMORY_SCOPE_SCHEMA,
+        requireScope: { type: 'boolean', description: 'Fail closed unless a complete four-field scope is supplied.' },
+        includeShared: { type: 'boolean', description: 'Include explicitly shared memories with scoped results. Defaults true.' },
       },
       required: ['toolName'],
     },
@@ -904,6 +922,10 @@ const TOOLS = [
           items: { type: 'string' },
           description: 'Optional protected-file globs that require explicit approval before editing or publishing',
         },
+        ttlMs: {
+          type: 'number',
+          description: 'Optional lease length in milliseconds. With it the scope becomes time-bounded authority (e.g. 90000 for "write under ./src for 90 seconds") and FAILS CLOSED on expiry: a lapsed lease authorises nothing until renewed. Omit for a permanent scope. Clamped to 60s..24h.',
+        },
         workflowContract: {
           type: 'object',
           description: 'Optional deterministic workflow run contract. Supports workflowId, allowedBranches, blockedActions, requiredEvidence, and completionGate.',
@@ -1047,6 +1069,16 @@ const TOOLS = [
     title: 'Record Verified Task Outcome',
     description: 'Record an idempotent task-level outcome with verification evidence, tool correctness, policy behavior, latency, cost, and business KPI movement. A completed response without evidence is recorded as not working.',
     inputSchema: TASK_OUTCOME_INPUT_SCHEMA,
+    outputSchema: {
+      type: 'object',
+      additionalProperties: false,
+      required: ['recorded', 'duplicate', 'receipt'],
+      properties: {
+        recorded: { type: 'boolean' },
+        duplicate: { type: 'boolean' },
+        receipt: { type: 'object', additionalProperties: true },
+      },
+    },
   }),
   readOnlyTool({
     name: 'get_task_outcomes',
@@ -1070,6 +1102,22 @@ const TOOLS = [
       additionalProperties: false,
       properties: {},
     },
+    outputSchema: {
+      type: 'object',
+      additionalProperties: true,
+      required: ['generatedAt', 'sampleSize', 'evidenceStatus', 'task', 'tools', 'safety', 'escalation', 'efficiency', 'businessOutcomes'],
+      properties: {
+        generatedAt: { type: 'string', format: 'date-time' },
+        sampleSize: { type: 'integer', minimum: 0 },
+        evidenceStatus: { type: 'string', enum: ['measured', 'insufficient_evidence'] },
+        task: { type: 'object', additionalProperties: true },
+        tools: { type: 'object', additionalProperties: true },
+        safety: { type: 'object', additionalProperties: true },
+        escalation: { type: 'object', additionalProperties: true },
+        efficiency: { type: 'object', additionalProperties: true },
+        businessOutcomes: { type: 'array' },
+      },
+    },
   }),
   destructiveTool({
     name: 'request_human_escalation',