thumbgate 1.29.2 → 1.30.0

package/scripts/parallel-workflow-orchestrator.js

@@ -1,18 +1,41 @@
 'use strict';
 
-const fs = require('fs');
-const path = require('path');
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
 const { getFeedbackPaths } = require('./feedback-loop');
 const { ensureDir } = require('./fs-utils');
 const { loadOptionalModule } = require('./private-core-boundary');
 
+const RUNNER_SCRIPT_PATH = path.join(__dirname, 'async-job-runner.js');
+
+function launchPublicManagedJob(jobSpec, options = {}) {
+  const publicRunner = require('./async-job-runner');
+  const jobId = options.jobId || jobSpec.id || `job_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+  const { jobDir } = publicRunner.getJobRuntimePaths(jobId);
+  ensureDir(jobDir);
+  const jobFilePath = path.join(jobDir, 'job.json');
+  const finalSpec = { ...jobSpec, id: jobId };
+  fs.writeFileSync(jobFilePath, `${JSON.stringify(finalSpec, null, 2)}\n`, 'utf8');
+  publicRunner.queueJob({ ...finalSpec, jobFilePath });
+  const child = spawn(process.execPath, [RUNNER_SCRIPT_PATH, `--run-file=${jobFilePath}`], {
+    cwd: options.cwd || process.cwd(),
+    env: process.env,
+    detached: true,
+    stdio: 'ignore',
+  });
+  child.unref();
+  return {
+    jobId,
+    jobFilePath,
+    launchMode: 'public-background',
+    pid: child.pid || null,
+  };
+}
+
 const launcher = loadOptionalModule(path.join(__dirname, 'hosted-job-launcher'), () => ({
-  launchManagedJob: () => {
-    throw new Error('Managed jobs require ThumbGate-Core.');
-  },
-  resumeHostedJob: () => {
-    throw new Error('Resuming hosted jobs requires ThumbGate-Core.');
-  },
+  launchManagedJob: launchPublicManagedJob,
 }));
 
 const runner = loadOptionalModule(path.join(__dirname, 'async-job-runner'), () => ({
@@ -20,8 +43,8 @@ const runner = loadOptionalModule(path.join(__dirname, 'async-job-runner'), () =
   listJobStates: () => [],
 }));
 
-const { launchManagedJob, resumeHostedJob } = launcher;
-const { readJobState, listJobStates } = runner;
+const { launchManagedJob } = launcher;
+const { readJobState } = runner;
 
 const DEFAULT_CONCURRENCY = 3;
 const POLL_INTERVAL_MS = 200;
@@ -45,7 +68,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'secret_scan',
-          command: 'node scripts/secret-scanner.js --json || true',
+          command: 'node scripts/secret-scanner.js --json',
         }
       ]
     });
@@ -55,7 +78,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'npm_audit',
-          command: 'npm audit --json || true',
+          command: 'npm audit --json',
         }
       ]
     });
@@ -65,7 +88,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'credential_gate_check',
-          command: 'node scripts/single-use-credential-gate.js plan || true',
+          command: 'node scripts/single-use-credential-gate.js plan',
         }
       ]
     });
@@ -76,7 +99,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'run_bench',
-          command: 'npx thumbgate bench --json --min-score=90 || true',
+          command: 'npx thumbgate bench --json --min-score=90',
         }
       ]
     });
@@ -86,7 +109,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'budget_status',
-          command: 'node scripts/budget-guard.js --status || true',
+          command: 'node scripts/budget-guard.js --status',
         }
       ]
     });
@@ -98,7 +121,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'search_fs',
-          command: 'node scripts/filesystem-search.js --query="pretool" --limit=5 || true',
+          command: 'node scripts/filesystem-search.js --query="pretool" --limit=5',
         }
       ]
     });
@@ -108,7 +131,7 @@ function planWorkflow(objective) {
       stages: [
         {
           name: 'ops_integrity',
-          command: 'node scripts/operational-integrity.js --ci || true',
+          command: 'node scripts/operational-integrity.js --ci',
         }
       ]
     });
@@ -119,7 +142,7 @@ function planWorkflow(objective) {
     plannedAt: nowIso(),
     subtasks: subtasks.map((task, idx) => ({
       ...task,
-      id: `subtask_${Date.now()}_${idx}_${Math.random().toString(36).slice(2, 6)}`,
+      id: `subtask_${Date.now()}_${idx}_${crypto.randomBytes(3).toString('hex')}`,
       autoImprove: false,
       verificationMode: 'none',
       recordFeedback: false,
@@ -132,12 +155,14 @@ function planWorkflow(objective) {
  * Polls active jobs until all complete, then consolidates the results.
  */
 async function executeWorkflow(objective, options = {}) {
-  const plan = planWorkflow(objective);
+  const plan = options.plan || planWorkflow(objective);
   const concurrency = Number(options.concurrency) || DEFAULT_CONCURRENCY;
   const timeoutMs = Number(options.timeoutMs) || 60000; // 60s timeout safety
+  const launchJob = options.launchManagedJob || launchManagedJob;
+  const getJobState = options.readJobState || readJobState;
 
   const { FEEDBACK_DIR } = getFeedbackPaths();
-  const workflowId = `wf_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+  const workflowId = `wf_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
   const workflowDir = path.join(FEEDBACK_DIR, 'workflows', workflowId);
   ensureDir(workflowDir);
 
@@ -145,17 +170,33 @@ async function executeWorkflow(objective, options = {}) {
   const queue = [...plan.subtasks];
   const results = [];
   const start = Date.now();
+  const statePath = path.join(workflowDir, 'state.json');
+
+  const persistState = (status) => {
+    fs.writeFileSync(statePath, `${JSON.stringify({
+      workflowId,
+      objective,
+      status,
+      updatedAt: nowIso(),
+      queue: queue.map((task) => ({ id: task.id, name: task.name })),
+      activeJobs: [...activeJobs.entries()].map(([taskId, info]) => ({ taskId, ...info })),
+      results,
+    }, null, 2)}\n`, 'utf8');
+  };
 
   const runNext = () => {
     while (activeJobs.size < concurrency && queue.length > 0) {
       const task = queue.shift();
-      const launched = launchManagedJob(task, { cwd: options.cwd });
+      const launched = launchJob(task, { cwd: options.cwd });
       activeJobs.set(task.id, {
         jobId: launched.jobId,
         taskName: task.name,
         launchedAt: Date.now(),
+        pid: launched.pid || null,
+        launchMode: launched.launchMode || 'managed',
       });
     }
+    persistState('running');
   };
 
   runNext();
@@ -166,7 +207,7 @@ async function executeWorkflow(objective, options = {}) {
       let allDone = true;
 
       for (const [taskId, info] of activeJobs.entries()) {
-        const jobState = readJobState(info.jobId);
+        const jobState = getJobState(info.jobId);
         if (!jobState) {
           allDone = false;
           continue;
@@ -193,11 +234,21 @@ async function executeWorkflow(objective, options = {}) {
       const elapsed = Date.now() - start;
       if (allDone && queue.length === 0) {
         clearInterval(interval);
+        persistState(results.every((result) => result.status === 'completed')
+          ? 'completed'
+          : 'completed_with_failures');
         resolve();
       } else if (elapsed >= timeoutMs) {
         clearInterval(interval);
         // Timeout remaining active tasks
         for (const [taskId, info] of activeJobs.entries()) {
+          if (info.pid) {
+            try {
+              process.kill(process.platform === 'win32' ? info.pid : -info.pid, 'SIGTERM');
+            } catch {
+              // The worker may have exited between the last poll and timeout.
+            }
+          }
           results.push({
             taskId,
             taskName: info.taskName,
@@ -206,6 +257,17 @@ async function executeWorkflow(objective, options = {}) {
             lastError: { message: `Subtask timed out after ${timeoutMs}ms`, code: 'TIMEOUT' },
           });
         }
+        for (const task of queue.splice(0)) {
+          results.push({
+            taskId: task.id,
+            taskName: task.name,
+            jobId: null,
+            status: 'timeout',
+            lastError: { message: `Subtask was not launched before ${timeoutMs}ms`, code: 'TIMEOUT' },
+          });
+        }
+        activeJobs.clear();
+        persistState('timed_out');
         resolve();
       }
     }, POLL_INTERVAL_MS);
@@ -234,6 +296,7 @@ async function executeWorkflow(objective, options = {}) {
     durationMs,
     reportPath,
     results,
+    statePath,
   };
 }
 
@@ -290,4 +353,5 @@ module.exports = {
   planWorkflow,
   executeWorkflow,
   compileWorkflowReport,
+  launchPublicManagedJob,
 };

package/scripts/published-cli.js

@@ -13,6 +13,14 @@ function runtimePrefixDir(prefixDir) {
   return prefixDir || path.join(os.homedir(), '.thumbgate', 'runtime');
 }
 
+// For GENERATED SHELL COMMANDS only. Shell command strings land in shared, committed config
+// (.mcp.json entries, hook command lines), so expanding os.homedir() at generation time bakes
+// the generating machine's home into files other machines execute — /Users/alice/.thumbgate
+// fails with a permission error on bob's machine. shellQuote uses double quotes, so a literal
+// $HOME expands at RUNTIME on whichever machine runs the command. Non-shell consumers
+// (execFileSync paths) must keep using runtimePrefixDir, which returns a real filesystem path.
+const SHELL_RUNTIME_PREFIX = '$HOME/.thumbgate/runtime';
+
 function installedRuntimeBin(prefixDir) {
   return path.join(runtimePrefixDir(prefixDir), 'node_modules', '.bin', 'thumbgate');
 }
@@ -32,7 +40,9 @@ function publishedCliArgs(pkgVersion, commandArgs = [], options = {}) {
 }
 
 function publishedCliShellCommand(pkgVersion, commandArgs = [], options = {}) {
-  const prefixDir = runtimePrefixDir(options.prefixDir);
+  // Default to the runtime-expanded $HOME form; an explicit options.prefixDir (tests,
+  // throwaway prefixes) is honoured verbatim.
+  const prefixDir = options.prefixDir || SHELL_RUNTIME_PREFIX;
   const runtimeBin = installedRuntimeBin(prefixDir);
   const escapedArgs = commandArgs.map(shellQuote).join(' ');
   const fastPath = `[ -x ${shellQuote(runtimeBin)} ] && exec ${shellQuote(runtimeBin)}${escapedArgs ? ` ${escapedArgs}` : ''}`;

package/scripts/refresh-proof-pack.js

@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * refresh-proof-pack.js — regenerate (or check) the public evaluation scorecard.
+ *
+ * Cadence (repo policy): GitHub Actions schedule is limited to CodeQL. Noncritical
+ * loops run via workflow_dispatch or local LaunchAgent:
+ *   npm run proof-pack:refresh          # write public/eval-scorecard.html
+ *   npm run proof-pack:refresh:check    # CI gate: metrics must still match
+ *   npm run proof-pack:schedule         # install daily local LaunchAgent
+ *
+ * Isolation: generation goes through generate-eval-scorecard → thumbgate-bench
+ * isolated runtime (strict enforcement pinned).
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const PROJECT_ROOT = path.resolve(__dirname, '..');
+const SCORECARD_HTML = path.join(PROJECT_ROOT, 'public', 'eval-scorecard.html');
+const SCORECARD_JSON = path.join(PROJECT_ROOT, 'public', 'eval-scorecard.json');
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    write: false,
+    check: false,
+    json: false,
+    help: false,
+    minScore: 90,
+  };
+  for (const arg of argv) {
+    if (arg === '--write') args.write = true;
+    else if (arg === '--check') args.check = true;
+    else if (arg === '--json') args.json = true;
+    else if (arg === '--help' || arg === '-h') args.help = true;
+    else if (arg.startsWith('--min-score=')) {
+      args.minScore = Number(arg.slice('--min-score='.length));
+    }
+  }
+  if (!args.write && !args.check) {
+    // Default to write for operator cadence runs.
+    args.write = true;
+  }
+  return args;
+}
+
+function printHelp() {
+  console.log(`Usage: node scripts/refresh-proof-pack.js [--write] [--check] [--json] [--min-score=90]
+
+  --write   Regenerate public/eval-scorecard.html (+ .json sidecar)
+  --check   Fail if committed scorecard metrics diverge from a fresh bench run
+  --json    Print machine-readable summary to stdout
+  --min-score=N  Minimum composite score (default 90)
+`);
+}
+
+function extractMetricsFromHtml(html) {
+  const metrics = {};
+  // Prefer JSON-LD Dataset variableMeasured
+  const ldMatch = html.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/);
+  if (ldMatch) {
+    try {
+      const ld = JSON.parse(ldMatch[1]);
+      const vars = Array.isArray(ld.variableMeasured) ? ld.variableMeasured : [];
+      for (const item of vars) {
+        if (item && item.name != null) metrics[item.name] = item.value;
+      }
+    } catch {
+      // fall through to regex
+    }
+  }
+  const scoreMatch = html.match(/composite score <strong>([^<]+)<\/strong>/i)
+    || html.match(/composite score[^0-9]*([0-9]+)/i);
+  if (scoreMatch && metrics.score == null) {
+    metrics.score = Number(scoreMatch[1]);
+  }
+  const passMatch = html.match(/Overall:\s*<span class="(good|bad)">(PASSED|FAILED)<\/span>/i);
+  if (passMatch) {
+    metrics.passedLabel = passMatch[2].toUpperCase();
+  }
+  return metrics;
+}
+
+function normalizeMetrics(metrics = {}) {
+  const keys = [
+    'score',
+    'taskSuccessRate',
+    'unsafeActionRate',
+    'blockedUnsafeRate',
+    'capabilityRate',
+    'falseBlockRate',
+    'replayStability',
+  ];
+  const out = {};
+  for (const key of keys) {
+    if (metrics[key] == null || metrics[key] === '') continue;
+    const n = Number(metrics[key]);
+    out[key] = Number.isFinite(n) ? Number(n.toFixed(4)) : metrics[key];
+  }
+  return out;
+}
+
+function metricsEqual(a, b, options = {}) {
+  const left = normalizeMetrics(a);
+  const right = normalizeMetrics(b);
+  // When comparing a committed HTML extract to a fresh bench report, only
+  // assert keys present on the committed side (JSON-LD may omit some rates).
+  const keys = options.keys
+    || (options.committedOnly
+      ? Object.keys(left)
+      : [...new Set([...Object.keys(left), ...Object.keys(right)])]);
+  const diffs = [];
+  for (const key of keys) {
+    if (left[key] !== right[key]) {
+      diffs.push({ key, committed: left[key], fresh: right[key] });
+    }
+  }
+  return { equal: diffs.length === 0, diffs, left, right };
+}
+
+function runFreshBench() {
+  const { generate, runBench } = require('./generate-eval-scorecard');
+  // Prefer direct bench for metrics; generate for write path.
+  let report;
+  try {
+    report = runBench();
+  } catch {
+    // generate also runs the bench
+    report = null;
+  }
+  return { generate, report };
+}
+
+function buildSidecar(report, version, nowIso) {
+  const metrics = report.metrics || report;
+  return {
+    generatedAt: nowIso,
+    version,
+    sourcePath: report.sourcePath || 'bench/thumbgate-bench.json',
+    passed: report.passed !== false,
+    isolatedRuntime: report.isolatedRuntime !== false,
+    metrics: normalizeMetrics(metrics),
+    scenarioCount: Array.isArray(report.scenarios) ? report.scenarios.length : null,
+    proofUrl: 'https://thumbgate.ai/eval-scorecard',
+  };
+}
+
+function refreshWrite(options = {}) {
+  const { generate } = require('./generate-eval-scorecard');
+  const now = options.now instanceof Date ? options.now : new Date();
+  const result = generate({
+    now,
+    outputPath: options.outputPath || SCORECARD_HTML,
+  });
+  const version = options.version || require(path.join(PROJECT_ROOT, 'package.json')).version;
+  const sidecar = buildSidecar(result.report, version, now.toISOString());
+  const sidecarPath = options.sidecarPath || SCORECARD_JSON;
+  fs.writeFileSync(sidecarPath, `${JSON.stringify(sidecar, null, 2)}\n`, 'utf8');
+  return {
+    mode: 'write',
+    htmlPath: result.outPath,
+    sidecarPath,
+    passed: result.report.passed !== false,
+    metrics: sidecar.metrics,
+    score: sidecar.metrics.score,
+  };
+}
+
+function refreshCheck(options = {}) {
+  const htmlPath = options.htmlPath || SCORECARD_HTML;
+  if (!fs.existsSync(htmlPath)) {
+    throw new Error(`Missing committed scorecard: ${htmlPath}`);
+  }
+  const committedHtml = fs.readFileSync(htmlPath, 'utf8');
+  const committed = normalizeMetrics(extractMetricsFromHtml(committedHtml));
+
+  const { runBench } = require('./generate-eval-scorecard');
+  const report = options.report || runBench();
+  const fresh = normalizeMetrics(report.metrics || report);
+  const comparison = metricsEqual(committed, fresh, { committedOnly: true });
+  const score = Number(fresh.score ?? committed.score);
+  const minScore = options.minScore ?? 90;
+  const scoreOk = Number.isFinite(score) && score >= minScore;
+  const passed = report.passed !== false && scoreOk && comparison.equal;
+
+  return {
+    mode: 'check',
+    passed,
+    scoreOk,
+    metricsMatch: comparison.equal,
+    diffs: comparison.diffs,
+    committed,
+    fresh,
+    minScore,
+    reportPassed: report.passed !== false,
+  };
+}
+
+function main(argv = process.argv.slice(2)) {
+  const args = parseArgs(argv);
+  if (args.help) {
+    printHelp();
+    return 0;
+  }
+
+  let summary;
+  if (args.check) {
+    summary = refreshCheck({ minScore: args.minScore });
+  } else {
+    summary = refreshWrite();
+    if (Number(summary.score) < args.minScore || summary.passed === false) {
+      summary.checkFailed = true;
+    }
+  }
+
+  if (args.json) {
+    console.log(JSON.stringify(summary, null, 2));
+  } else if (summary.mode === 'write') {
+    console.log(
+      `Proof pack scorecard written: ${summary.htmlPath} (score=${summary.score}, passed=${summary.passed})`,
+    );
+    console.log(`Sidecar: ${summary.sidecarPath}`);
+  } else {
+    console.log(
+      `Proof pack check: metricsMatch=${summary.metricsMatch} scoreOk=${summary.scoreOk} reportPassed=${summary.reportPassed}`,
+    );
+    if (summary.diffs.length) {
+      for (const diff of summary.diffs) {
+        console.log(`  drift ${diff.key}: committed=${diff.committed} fresh=${diff.fresh}`);
+      }
+    }
+  }
+
+  if (summary.mode === 'check' && !summary.passed) return 1;
+  if (summary.mode === 'write' && summary.checkFailed) return 1;
+  return 0;
+}
+
+if (path.resolve(process.argv[1] || '') === path.resolve(__filename)) {
+  try {
+    process.exitCode = main();
+  } catch (err) {
+    console.error(err.message || err);
+    process.exitCode = 1;
+  }
+}
+
+module.exports = {
+  parseArgs,
+  extractMetricsFromHtml,
+  normalizeMetrics,
+  metricsEqual,
+  refreshWrite,
+  refreshCheck,
+  buildSidecar,
+  main,
+  SCORECARD_HTML,
+  SCORECARD_JSON,
+};