thumbgate 1.29.1 → 1.30.0

package/scripts/model-eval.js

@@ -0,0 +1,308 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * model-eval.js — honest evaluation primitives for ThumbGate's learned models.
+ *
+ * WHY THIS EXISTS
+ *
+ * The risk scorer reported exactly one number: `trainingAccuracy`, measured on the same rows
+ * it trained on. On 2026-07-28 that number was 0.820 — which sounds good until you notice the
+ * base rate was 0.711. A model that answers "high-risk" unconditionally scores 71.1%, so the
+ * headline figure was measuring an 11-point in-sample lift and presenting it as quality.
+ * Nothing anywhere compared against that trivial baseline, and no split existed, so the
+ * generalization number was not merely bad — it was unknown.
+ *
+ * Accuracy is also the wrong summary for a 71/29 split. These primitives therefore report the
+ * metrics that survive class imbalance (precision/recall/F1/MCC/ROC-AUC) and the ones that say
+ * whether a probability means anything (Brier score, expected calibration error).
+ *
+ * DETERMINISM IS A REQUIREMENT, NOT A PREFERENCE.
+ * Splits are derived from a content hash, never from Math.random(). The same corpus must
+ * produce the same split on every machine and every run, or a "quality regression" is
+ * indistinguishable from a reshuffle and the CI gate built on top is noise.
+ *
+ * Everything here is pure: no I/O, no clock, no global state. That is what makes it testable.
+ */
+
+/** FNV-1a. Small, fast, and stable across platforms — the only properties we need. */
+function hashString(text) {
+  let hash = 0x811c9dc5;
+  const value = String(text);
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    // 32-bit FNV prime multiply via shifts; Math.imul keeps this exact in JS.
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return hash >>> 0;
+}
+
+/** Labels arrive as 1/-1 (AdaBoost) or 1/0 (everything else). Normalize to 1/0. */
+function toBinaryLabel(label) {
+  return Number(label) === 1 ? 1 : 0;
+}
+
+/**
+ * Deterministic stratified split.
+ *
+ * Stratified because a random split of a 71/29 corpus can hand back a test fold with a
+ * materially different base rate, which moves the very number we are trying to measure.
+ * Splitting inside each class keeps the test fold's base rate close to the corpus.
+ *
+ * Returns { train, test }. Both are always non-empty when the input supports it; when a class
+ * is too small to contribute a test row, the split degrades to "everything is training" rather
+ * than silently producing a test fold with one class in it (an AUC of NaN dressed up as data).
+ */
+function stratifiedSplit(examples, options = {}) {
+  const testFraction = Number(options.testFraction || 0.25);
+  const keyFn = options.keyFn || ((example, index) => JSON.stringify(example.features || example) + index);
+
+  // Group ACROSS classes, then assign whole groups — StratifiedGroupKFold semantics.
+  //
+  // Two bugs led here, both caught by tests/risk-model-quality.test.js:
+  //   1. Assigning individual rows split tied blocks, so an identical row could sit in both
+  //      folds and the test fold scored rows the model had memorized.
+  //   2. Grouping per class still split duplicates, because with label noise the SAME feature
+  //      vector can carry both labels — the two copies then landed in different class buckets
+  //      and different folds.
+  // Grouping globally is the only version where "this input is in exactly one fold" holds.
+  const classTotals = new Map();
+  const groups = new Map();
+  examples.forEach((example, index) => {
+    const label = toBinaryLabel(example.label);
+    classTotals.set(label, (classTotals.get(label) || 0) + 1);
+    const hash = hashString(keyFn(example, index));
+    if (!groups.has(hash)) groups.set(hash, { members: [], counts: new Map() });
+    const group = groups.get(hash);
+    group.members.push(example);
+    group.counts.set(label, (group.counts.get(label) || 0) + 1);
+  });
+
+  // A single-class corpus has nothing to measure; there is no honest split of it.
+  if (classTotals.size < 2) return { train: examples.slice(), test: [] };
+
+  // Per-class quotas keep the test fold's base rate near the corpus even though whole groups
+  // move together. Without quotas, one large group could swamp the fold and shift the very
+  // base rate the lift is measured against.
+  const quotas = new Map();
+  for (const [label, total] of classTotals) quotas.set(label, Math.floor(total * testFraction));
+
+  const train = [];
+  const test = [];
+  const taken = new Map();
+  // Order by hash, not by position: input order in a JSONL log is chronological, so a
+  // positional split would put all recent rows in one fold and measure drift, not skill.
+  const ordered = [...groups.entries()].sort((left, right) => left[0] - right[0]);
+
+  for (const [, group] of ordered) {
+    // Take the group only if it fits ENTIRELY within the remaining quota of every class it
+    // contains. Checking merely "is this class still under quota" admitted the whole group and
+    // let it overshoot by hundreds of rows — the real corpus has a 655-row content group, so a
+    // nominal 25% fold could be swamped by one category and the base rate the lift is measured
+    // against would shift underneath it.
+    let fits = true;
+    for (const [label, count] of group.counts) {
+      if ((taken.get(label) || 0) + count > (quotas.get(label) || 0)) { fits = false; break; }
+    }
+    if (fits) {
+      test.push(...group.members);
+      for (const [label, count] of group.counts) taken.set(label, (taken.get(label) || 0) + count);
+    } else {
+      train.push(...group.members);
+    }
+  }
+
+  // Both classes must appear in the test fold. With a small minority class the floor()'d quota
+  // can be 0, which would leave every minority group in training and produce a single-class
+  // test fold: ROC-AUC is undefined there and precision/recall/MCC are degenerate, yet the
+  // report would still be marked available. Refusing to split is the honest outcome.
+  const testClasses = new Set(test.map((example) => toBinaryLabel(example.label)));
+  if (testClasses.size < 2) return { train: examples.slice(), test: [] };
+
+  // If either side collapsed, report no test fold instead of a meaningless one.
+  if (test.length === 0 || train.length === 0) return { train: examples.slice(), test: [] };
+  return { train, test };
+}
+
+/**
+ * Threshold metrics. `pairs` is [{ probability, label }].
+ * Precision/recall are reported for the positive (high-risk) class.
+ */
+function classificationMetrics(pairs, threshold = 0.5) {
+  let truePositive = 0;
+  let falsePositive = 0;
+  let trueNegative = 0;
+  let falseNegative = 0;
+
+  for (const pair of pairs) {
+    const actual = toBinaryLabel(pair.label);
+    const predicted = Number(pair.probability) >= threshold ? 1 : 0;
+    if (predicted === 1 && actual === 1) truePositive += 1;
+    else if (predicted === 1 && actual === 0) falsePositive += 1;
+    else if (predicted === 0 && actual === 0) trueNegative += 1;
+    else falseNegative += 1;
+  }
+
+  const total = pairs.length || 1;
+  const precision = truePositive + falsePositive > 0 ? truePositive / (truePositive + falsePositive) : 0;
+  const recall = truePositive + falseNegative > 0 ? truePositive / (truePositive + falseNegative) : 0;
+  const specificity = trueNegative + falsePositive > 0 ? trueNegative / (trueNegative + falsePositive) : 0;
+  const f1 = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+
+  // Matthews correlation: the summary that does not flatter a majority-class predictor.
+  // A constant classifier scores 0 here no matter how skewed the corpus is.
+  const mccDenominator = Math.sqrt(
+    (truePositive + falsePositive)
+    * (truePositive + falseNegative)
+    * (trueNegative + falsePositive)
+    * (trueNegative + falseNegative),
+  );
+  const mcc = mccDenominator > 0
+    ? ((truePositive * trueNegative) - (falsePositive * falseNegative)) / mccDenominator
+    : 0;
+
+  return {
+    accuracy: (truePositive + trueNegative) / total,
+    precision,
+    recall,
+    specificity,
+    f1,
+    mcc,
+    confusion: { truePositive, falsePositive, trueNegative, falseNegative },
+  };
+}
+
+/**
+ * ROC-AUC by the rank method, with tied scores sharing an average rank.
+ *
+ * Tie handling matters here specifically: a stump ensemble emits a small set of discrete
+ * scores, so ties are the common case rather than an edge case. Treating them by input order
+ * would make AUC depend on row order — a number that changes when you sort your log.
+ */
+function rocAuc(pairs) {
+  const positives = pairs.filter((pair) => toBinaryLabel(pair.label) === 1).length;
+  const negatives = pairs.length - positives;
+  if (positives === 0 || negatives === 0) return null; // undefined, and saying so beats guessing
+
+  const ordered = pairs
+    .map((pair) => ({ score: Number(pair.probability), label: toBinaryLabel(pair.label) }))
+    .sort((left, right) => left.score - right.score);
+
+  const ranks = new Array(ordered.length);
+  let index = 0;
+  while (index < ordered.length) {
+    let end = index;
+    while (end + 1 < ordered.length && ordered[end + 1].score === ordered[index].score) end += 1;
+    // Ranks are 1-based; tied entries all take the midpoint of the block they occupy.
+    const averageRank = ((index + 1) + (end + 1)) / 2;
+    for (let position = index; position <= end; position += 1) ranks[position] = averageRank;
+    index = end + 1;
+  }
+
+  let positiveRankSum = 0;
+  ordered.forEach((entry, position) => {
+    if (entry.label === 1) positiveRankSum += ranks[position];
+  });
+
+  return (positiveRankSum - (positives * (positives + 1)) / 2) / (positives * negatives);
+}
+
+/** Mean squared error of the probability itself. Rewards being uncertain when uncertain. */
+function brierScore(pairs) {
+  if (pairs.length === 0) return null;
+  const total = pairs.reduce((sum, pair) => {
+    const actual = toBinaryLabel(pair.label);
+    const probability = Number(pair.probability);
+    return sum + ((probability - actual) ** 2);
+  }, 0);
+  return total / pairs.length;
+}
+
+/**
+ * Expected calibration error: |predicted - observed| averaged over equal-width bins,
+ * weighted by bin population.
+ *
+ * This is the metric that catches a model whose ranking is fine but whose probabilities are
+ * meaningless — which matters because downstream gates threshold on the probability, not on
+ * the rank.
+ */
+function calibration(pairs, binCount = 10) {
+  if (pairs.length === 0) return { expectedCalibrationError: null, bins: [] };
+  const bins = Array.from({ length: binCount }, () => ({ count: 0, predicted: 0, observed: 0 }));
+
+  for (const pair of pairs) {
+    const probability = Math.min(0.999999, Math.max(0, Number(pair.probability)));
+    const slot = Math.min(binCount - 1, Math.floor(probability * binCount));
+    bins[slot].count += 1;
+    bins[slot].predicted += probability;
+    bins[slot].observed += toBinaryLabel(pair.label);
+  }
+
+  let error = 0;
+  const report = bins.map((bin, slot) => {
+    if (bin.count === 0) return { bin: slot, count: 0, meanPredicted: null, meanObserved: null };
+    const meanPredicted = bin.predicted / bin.count;
+    const meanObserved = bin.observed / bin.count;
+    error += (bin.count / pairs.length) * Math.abs(meanPredicted - meanObserved);
+    return { bin: slot, count: bin.count, meanPredicted, meanObserved };
+  });
+
+  return { expectedCalibrationError: error, bins: report };
+}
+
+/**
+ * Full report for a set of (probability, label) pairs.
+ *
+ * `baselineAccuracy` and `lift` are the point of this whole module: a model is only worth
+ * running if it beats the constant classifier, and that comparison must appear next to the
+ * accuracy every single time so it can never again be quoted without it.
+ */
+function evaluate(pairs, options = {}) {
+  const threshold = Number(options.threshold ?? 0.5);
+  const positives = pairs.filter((pair) => toBinaryLabel(pair.label) === 1).length;
+  const baseRate = pairs.length > 0 ? positives / pairs.length : 0;
+  // The trivial classifier always answers with the majority class.
+  const baselineAccuracy = Math.max(baseRate, 1 - baseRate);
+  const metrics = classificationMetrics(pairs, threshold);
+
+  return {
+    sampleCount: pairs.length,
+    positiveCount: positives,
+    baseRate,
+    baselineAccuracy,
+    accuracy: metrics.accuracy,
+    lift: metrics.accuracy - baselineAccuracy,
+    precision: metrics.precision,
+    recall: metrics.recall,
+    specificity: metrics.specificity,
+    f1: metrics.f1,
+    mcc: metrics.mcc,
+    rocAuc: rocAuc(pairs),
+    brierScore: brierScore(pairs),
+    expectedCalibrationError: calibration(pairs, options.calibrationBins || 10).expectedCalibrationError,
+    confusion: metrics.confusion,
+  };
+}
+
+/** Round every float in a report so persisted artifacts diff cleanly. */
+function roundReport(report, digits = 4) {
+  const factor = 10 ** digits;
+  const rounded = {};
+  for (const [key, value] of Object.entries(report || {})) {
+    if (typeof value === 'number' && Number.isFinite(value)) rounded[key] = Math.round(value * factor) / factor;
+    else if (value && typeof value === 'object' && !Array.isArray(value)) rounded[key] = roundReport(value, digits);
+    else rounded[key] = value;
+  }
+  return rounded;
+}
+
+module.exports = {
+  hashString,
+  stratifiedSplit,
+  classificationMetrics,
+  rocAuc,
+  brierScore,
+  calibration,
+  evaluate,
+  roundReport,
+};

package/scripts/parallel-workflow-orchestrator.js

@@ -1,18 +1,41 @@
 'use strict';
 
-const fs = require('fs');
-const path = require('path');
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
 const { getFeedbackPaths } = require('./feedback-loop');
 const { ensureDir } = require('./fs-utils');
 const { loadOptionalModule } = require('./private-core-boundary');
 
+const RUNNER_SCRIPT_PATH = path.join(__dirname, 'async-job-runner.js');
+
+function launchPublicManagedJob(jobSpec, options = {}) {
+  const publicRunner = require('./async-job-runner');
+  const jobId = options.jobId || jobSpec.id || `job_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+  const { jobDir } = publicRunner.getJobRuntimePaths(jobId);
+  ensureDir(jobDir);
+  const jobFilePath = path.join(jobDir, 'job.json');
+  const finalSpec = { ...jobSpec, id: jobId };
+  fs.writeFileSync(jobFilePath, `${JSON.stringify(finalSpec, null, 2)}\n`, 'utf8');
+  publicRunner.queueJob({ ...finalSpec, jobFilePath });
+  const child = spawn(process.execPath, [RUNNER_SCRIPT_PATH, `--run-file=${jobFilePath}`], {
+    cwd: options.cwd || process.cwd(),
+    env: process.env,
+    detached: true,
+    stdio: 'ignore',
+  });
+  child.unref();
+  return {
+    jobId,
+    jobFilePath,
+    launchMode: 'public-background',
+    pid: child.pid || null,
+  };
+}
+
 const launcher = loadOptionalModule(path.join(__dirname, 'hosted-job-launcher'), () => ({
-  launchManagedJob: () => {
-    throw new Error('Managed jobs require ThumbGate-Core.');
-  },
-  resumeHostedJob: () => {
-    throw new Error('Resuming hosted jobs requires ThumbGate-Core.');
-  },
+  launchManagedJob: launchPublicManagedJob,
 }));
 
 const runner = loadOptionalModule(path.join(__dirname, 'async-job-runner'), () => ({
@@ -20,8 +43,8 @@ const runner = loadOptionalModule(path.join(__dirname, 'async-job-runner'), () =
   listJobStates: () => [],
 }));
 
-const { launchManagedJob, resumeHostedJob } = launcher;
-const { readJobState, listJobStates } = runner;
+const { launchManagedJob } = launcher;
+const { readJobState } = runner;
 
 const DEFAULT_CONCURRENCY = 3;
 const POLL_INTERVAL_MS = 200;
@@ -45,7 +68,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'secret_scan',
-          command: 'node scripts/secret-scanner.js --json || true',
+          command: 'node scripts/secret-scanner.js --json',
         }
       ]
     });
@@ -55,7 +78,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'npm_audit',
-          command: 'npm audit --json || true',
+          command: 'npm audit --json',
         }
       ]
     });
@@ -65,7 +88,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'credential_gate_check',
-          command: 'node scripts/single-use-credential-gate.js plan || true',
+          command: 'node scripts/single-use-credential-gate.js plan',
         }
       ]
     });
@@ -76,7 +99,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'run_bench',
-          command: 'npx thumbgate bench --json --min-score=90 || true',
+          command: 'npx thumbgate bench --json --min-score=90',
         }
       ]
     });
@@ -86,7 +109,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'budget_status',
-          command: 'node scripts/budget-guard.js --status || true',
+          command: 'node scripts/budget-guard.js --status',
         }
       ]
     });
@@ -98,7 +121,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'search_fs',
-          command: 'node scripts/filesystem-search.js --query="pretool" --limit=5 || true',
+          command: 'node scripts/filesystem-search.js --query="pretool" --limit=5',
         }
       ]
     });
@@ -108,7 +131,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'ops_integrity',
-          command: 'node scripts/operational-integrity.js --ci || true',
+          command: 'node scripts/operational-integrity.js --ci',
         }
       ]
     });
@@ -119,7 +142,7 @@ function planWorkflow(objective) {
     plannedAt: nowIso(),
     subtasks: subtasks.map((task, idx) => ({
       ...task,
-      id: `subtask_${Date.now()}_${idx}_${Math.random().toString(36).slice(2, 6)}`,
+      id: `subtask_${Date.now()}_${idx}_${crypto.randomBytes(3).toString('hex')}`,
       autoImprove: false,
       verificationMode: 'none',
       recordFeedback: false,
@@ -132,12 +155,14 @@ function planWorkflow(objective) {
  * Polls active jobs until all complete, then consolidates the results.
  */
 async function executeWorkflow(objective, options = {}) {
-  const plan = planWorkflow(objective);
+  const plan = options.plan || planWorkflow(objective);
   const concurrency = Number(options.concurrency) || DEFAULT_CONCURRENCY;
   const timeoutMs = Number(options.timeoutMs) || 60000; // 60s timeout safety
+  const launchJob = options.launchManagedJob || launchManagedJob;
+  const getJobState = options.readJobState || readJobState;
 
   const { FEEDBACK_DIR } = getFeedbackPaths();
-  const workflowId = `wf_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+  const workflowId = `wf_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
   const workflowDir = path.join(FEEDBACK_DIR, 'workflows', workflowId);
   ensureDir(workflowDir);
 
@@ -145,17 +170,33 @@ async function executeWorkflow(objective, options = {}) {
   const queue = [...plan.subtasks];
   const results = [];
   const start = Date.now();
+  const statePath = path.join(workflowDir, 'state.json');
+
+  const persistState = (status) => {
+    fs.writeFileSync(statePath, `${JSON.stringify({
+      workflowId,
+      objective,
+      status,
+      updatedAt: nowIso(),
+      queue: queue.map((task) => ({ id: task.id, name: task.name })),
+      activeJobs: [...activeJobs.entries()].map(([taskId, info]) => ({ taskId, ...info })),
+      results,
+    }, null, 2)}\n`, 'utf8');
+  };
 
   const runNext = () => {
     while (activeJobs.size < concurrency && queue.length > 0) {
       const task = queue.shift();
-      const launched = launchManagedJob(task, { cwd: options.cwd });
+      const launched = launchJob(task, { cwd: options.cwd });
       activeJobs.set(task.id, {
         jobId: launched.jobId,
         taskName: task.name,
         launchedAt: Date.now(),
+        pid: launched.pid || null,
+        launchMode: launched.launchMode || 'managed',
       });
     }
+    persistState('running');
   };
 
   runNext();
@@ -166,7 +207,7 @@ async function executeWorkflow(objective, options = {}) {
       let allDone = true;
 
       for (const [taskId, info] of activeJobs.entries()) {
-        const jobState = readJobState(info.jobId);
+        const jobState = getJobState(info.jobId);
         if (!jobState) {
           allDone = false;
           continue;
@@ -193,11 +234,21 @@ async function executeWorkflow(objective, options = {}) {
       const elapsed = Date.now() - start;
       if (allDone && queue.length === 0) {
         clearInterval(interval);
+        persistState(results.every((result) => result.status === 'completed')
+          ? 'completed'
+          : 'completed_with_failures');
         resolve();
       } else if (elapsed >= timeoutMs) {
         clearInterval(interval);
         // Timeout remaining active tasks
         for (const [taskId, info] of activeJobs.entries()) {
+          if (info.pid) {
+            try {
+              process.kill(process.platform === 'win32' ? info.pid : -info.pid, 'SIGTERM');
+            } catch {
+              // The worker may have exited between the last poll and timeout.
+            }
+          }
           results.push({
             taskId,
             taskName: info.taskName,
@@ -206,6 +257,17 @@ async function executeWorkflow(objective, options = {}) {
             lastError: { message: `Subtask timed out after ${timeoutMs}ms`, code: 'TIMEOUT' },
           });
         }
+        for (const task of queue.splice(0)) {
+          results.push({
+            taskId: task.id,
+            taskName: task.name,
+            jobId: null,
+            status: 'timeout',
+            lastError: { message: `Subtask was not launched before ${timeoutMs}ms`, code: 'TIMEOUT' },
+          });
+        }
+        activeJobs.clear();
+        persistState('timed_out');
         resolve();
       }
     }, POLL_INTERVAL_MS);
@@ -234,6 +296,7 @@ async function executeWorkflow(objective, options = {}) {
     durationMs,
     reportPath,
     results,
+    statePath,
   };
 }
 
@@ -290,4 +353,5 @@ module.exports = {
   planWorkflow,
   executeWorkflow,
   compileWorkflowReport,
+  launchPublicManagedJob,
 };

package/scripts/prompt-eval.js

@@ -143,6 +143,8 @@ function handleRejectExpectation(checks, result, expected) {
 
   const wasRejected = result.accepted === false
     || result.status === 'rejected'
+    || result.status === 'clarification_required'
+    || result.needsClarification === true
     || result.actionType === 'no-action';
   checks.push({
     criterion: 'shouldReject',
@@ -466,8 +468,9 @@ function runSuiteObject(suite, options = {}) {
   const skipped = results.filter((r) => r.status === 'skip').length;
   const totalScore = results.length > 0
     ? Math.round(results.reduce((s, r) => s + r.score, 0) / results.length)
-    : 100;
+    : 0;
   const minScore = options.minScore ?? 80;
+  const insufficientEvidence = results.length === 0;
 
   return {
     suite: suite.name,
@@ -478,8 +481,9 @@ function runSuiteObject(suite, options = {}) {
     skipped,
     score: totalScore,
     minScore,
-    pass: totalScore >= minScore,
-    noCases: results.length === 0,
+    pass: !insufficientEvidence && totalScore >= minScore,
+    noCases: insufficientEvidence,
+    evidenceStatus: insufficientEvidence ? 'insufficient_evidence' : 'measured',
     feedbackDerived: suite.source && suite.source.type === 'feedback-log',
     generatedAt: new Date().toISOString(),
     results,
@@ -685,6 +689,7 @@ function runSuite(suitePath = DEFAULT_SUITE, options = {}) {
 
 function compareReports(currentReport, baselineReport) {
   const baselineById = new Map((baselineReport?.results || []).map((result) => [result.id, result]));
+  const currentById = new Map((currentReport?.results || []).map((result) => [result.id, result]));
   const regressions = [];
   const improvements = [];
 
@@ -717,12 +722,83 @@ function compareReports(currentReport, baselineReport) {
     }
   }
 
+  for (const baseline of baselineReport?.results || []) {
+    if (currentById.has(baseline.id)) continue;
+    regressions.push({
+      id: baseline.id,
+      baselineScore: baseline.score,
+      currentScore: null,
+      delta: null,
+      baselineStatus: baseline.status,
+      currentStatus: 'missing',
+    });
+  }
+
   return {
     baselineSuite: baselineReport?.suite || null,
     baselineScore: Number.isFinite(Number(baselineReport?.score)) ? Number(baselineReport.score) : null,
     scoreDelta: Number.isFinite(Number(baselineReport?.score)) ? currentReport.score - Number(baselineReport.score) : null,
     regressions,
     improvements,
+    baselineCases: baselineById.size,
+    currentCases: currentById.size,
+    baselineCoverageRate: baselineById.size
+      ? Math.round((Array.from(baselineById.keys()).filter((id) => currentById.has(id)).length / baselineById.size) * 10000) / 10000
+      : null,
+  };
+}
+
+function logSafeResult(result, index) {
+  const allowedStatuses = new Set(['pass', 'fail', 'error', 'skip']);
+  return {
+    case: index + 1,
+    status: allowedStatuses.has(result.status) ? result.status : 'error',
+    score: Number(result.score || 0),
+    passCount: Number(result.passCount || 0),
+    totalChecks: Number(result.totalChecks || 0),
+  };
+}
+
+function logSafeReport(report, suite, fromFeedback) {
+  const source = suite?.source || {};
+  const comparison = report.comparison
+    ? {
+        baselineScore: Number(report.comparison.baselineScore || 0),
+        scoreDelta: Number(report.comparison.scoreDelta || 0),
+        regressionCount: Number(report.comparison.regressions?.length || 0),
+        improvementCount: Number(report.comparison.improvements?.length || 0),
+        baselineCases: Number(report.comparison.baselineCases || 0),
+        currentCases: Number(report.comparison.currentCases || 0),
+        baselineCoverageRate: report.comparison.baselineCoverageRate,
+      }
+    : undefined;
+  return {
+    suiteType: fromFeedback ? 'feedback-derived' : 'configured',
+    total: Number(report.total || 0),
+    passed: Number(report.passed || 0),
+    failed: Number(report.failed || 0),
+    errors: Number(report.errors || 0),
+    skipped: Number(report.skipped || 0),
+    score: Number(report.score || 0),
+    minScore: Number(report.minScore || 0),
+    pass: report.pass === true,
+    noCases: report.noCases === true,
+    evidenceStatus: report.evidenceStatus === 'measured' ? 'measured' : 'insufficient_evidence',
+    feedbackDerived: report.feedbackDerived === true,
+    syntheticCount: Number(report.syntheticCount || 0),
+    comparison,
+    results: (report.results || []).map(logSafeResult),
+    suiteDefinition: fromFeedback
+      ? {
+          version: Number(suite?.version || 0),
+          source: {
+            type: 'feedback-log',
+            totalEntries: Number(source.totalEntries || 0),
+            selectedCases: Number(source.selectedCases || 0),
+          },
+          evaluationCount: Number(suite?.evaluations?.length || 0),
+        }
+      : undefined,
   };
 }
 
@@ -847,7 +923,7 @@ if (isCliInvocation()) {
   }
 
   if (json) {
-    console.log(JSON.stringify({ ...report, suiteDefinition: fromFeedback ? suite : undefined }, null, 2));
+    console.log(JSON.stringify(logSafeReport(report, suite, fromFeedback), null, 2));
   } else {
     console.log(`\n${report.suite}`);
     console.log('='.repeat(50));
@@ -883,6 +959,7 @@ module.exports = {
   gradeOutput,
   loadSuite,
   loadReport,
+  logSafeReport,
   compareReports,
   readJsonl,
   runEvaluation,