thumbgate 1.4.2 → 1.4.4

package/scripts/prove-data-quality.js

@@ -1,227 +0,0 @@
-'use strict';
-/**
- * Phase 7: Data Quality — Proof Gate
- *
- * Validates all QUAL-01 through QUAL-04 requirements offline.
- * Mirrors the pattern of prove-attribution.js (mkdtempSync + env override + execSync).
- *
- * Usage:
- *   node scripts/prove-data-quality.js
- *
- * Produces:
- *   proof/data-quality-report.json
- *   proof/data-quality-report.md
- */
-
-const { execSync } = require('child_process');
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-
-const ROOT = path.join(__dirname, '..');
-
-function resolveProofPaths() {
-  const proofDir = process.env.THUMBGATE_PROOF_DIR || path.join(ROOT, 'proof');
-  return {
-    proofDir,
-    reportJson: path.join(proofDir, 'data-quality-report.json'),
-    reportMd: path.join(proofDir, 'data-quality-report.md'),
-  };
-}
-
-function run() {
-  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-qual-proof-'));
-  const results = { passed: 0, failed: 0, requirements: {} };
-  const { proofDir, reportJson, reportMd } = resolveProofPaths();
-
-  const checks = [
-    {
-      id: 'QUAL-01',
-      desc: 'validate-feedback.js exports validateEntry with 4-level pipeline',
-      fn: () => {
-        delete require.cache[require.resolve('./validate-feedback')];
-        const v = require('./validate-feedback');
-        if (typeof v.validateEntry !== 'function') throw new Error('validateEntry not exported');
-        if (typeof v.validateSchema !== 'function') throw new Error('validateSchema not exported');
-        if (typeof v.validateSemantics !== 'function') throw new Error('validateSemantics not exported');
-        if (typeof v.detectAnomalies !== 'function') throw new Error('detectAnomalies not exported');
-        if (typeof v.generateCorrections !== 'function') throw new Error('generateCorrections not exported');
-
-        // Verify semantic inconsistency is caught
-        const r = v.validateEntry({
-          id: 'proof-x',
-          timestamp: new Date().toISOString(),
-          signal: 'positive',
-          reward: -1,
-          context: 'good work done',
-        });
-        if (r.valid) throw new Error('Expected invalid for positive+negative-reward');
-        if (!r.corrections.length) throw new Error('Expected auto-correction for reward');
-        if (r.correctedEntry.reward !== 1) throw new Error('Expected corrected reward=1');
-
-        // Verify sensitive data detection
-        const r2 = v.validateEntry({
-          id: 'proof-y',
-          timestamp: new Date().toISOString(),
-          signal: 'positive',
-          reward: 1,
-          context: 'api_key=abc123 was in the response',
-        });
-        if (!r2.issues.some((i) => i.type === 'security')) {
-          throw new Error('Expected security issue for api_key pattern');
-        }
-      },
-    },
-    {
-      id: 'QUAL-02',
-      desc: 'captureFeedback produces richContext with domain, filePaths, errorType, outcomeCategory',
-      fn: () => {
-        process.env.THUMBGATE_FEEDBACK_DIR = tmpDir;
-        // Clear module cache so env var takes effect
-        [
-          './feedback-loop',
-          './feedback-attribution',
-          './rlaif-self-audit',
-        ].forEach((m) => {
-          try {
-            delete require.cache[require.resolve(m)];
-          } catch {
-            // optional module
-          }
-        });
-        const { captureFeedback } = require('./feedback-loop');
-        const r = captureFeedback({
-          signal: 'positive',
-          context: 'unit tests added for edge cases',
-          tags: ['testing'],
-          filePaths: ['src/api.js'],
-        });
-        if (!r.feedbackEvent) throw new Error('No feedbackEvent in result');
-        const rc = r.feedbackEvent.richContext;
-        if (!rc) throw new Error('richContext missing from feedbackEvent');
-        if (typeof rc.domain !== 'string') throw new Error('richContext.domain must be string');
-        if (!Array.isArray(rc.filePaths)) throw new Error('richContext.filePaths must be array');
-        if (!('errorType' in rc)) throw new Error('richContext.errorType field missing');
-        if (typeof rc.outcomeCategory !== 'string') throw new Error('richContext.outcomeCategory must be string');
-        if (rc.domain !== 'testing') throw new Error(`Expected domain=testing, got ${rc.domain}`);
-      },
-    },
-    {
-      id: 'QUAL-03',
-      desc: 'inferOutcome returns granular categories beyond binary up/down',
-      fn: () => {
-        [
-          './feedback-loop',
-        ].forEach((m) => {
-          try { delete require.cache[require.resolve(m)]; } catch {}
-        });
-        const { inferOutcome } = require('./feedback-loop');
-        if (typeof inferOutcome !== 'function') throw new Error('inferOutcome not exported from feedback-loop');
-
-        const cases = [
-          ['positive', 'solved it first try', 'quick-success'],
-          ['positive', 'thorough comprehensive analysis', 'deep-success'],
-          ['positive', 'worked well overall', 'standard-success'],
-          ['negative', 'gave wrong incorrect answer', 'factual-error'],
-          ['negative', 'shallow surface level response', 'insufficient-depth'],
-          ['negative', 'guessed without checking docs', 'false-assumption'],
-        ];
-
-        for (const [signal, context, expected] of cases) {
-          const got = inferOutcome(signal, context);
-          if (got !== expected) {
-            throw new Error(`inferOutcome('${signal}', '${context}') = '${got}', expected '${expected}'`);
-          }
-        }
-      },
-    },
-    {
-      id: 'QUAL-04',
-      desc: 'test:quality (node --test tests/validate-feedback.test.js) passes with 0 failures',
-      fn: () => {
-        const out = execSync('node --test tests/validate-feedback.test.js', {
-          cwd: ROOT,
-          env: { ...process.env, THUMBGATE_FEEDBACK_DIR: tmpDir },
-          encoding: 'utf8',
-          stdio: 'pipe',
-        });
-        // node:test exits non-zero on failure — if we get here, all tests passed
-        const failMatch = out.match(/ℹ fail (\d+)/);
-        if (failMatch && parseInt(failMatch[1], 10) > 0) {
-          throw new Error(`Tests failed: ${failMatch[1]} failure(s)\n${out.slice(-500)}`);
-        }
-      },
-    },
-  ];
-
-  console.log('Phase 7: Data Quality — Proof Gate\n');
-  console.log('Checking requirements:\n');
-
-  for (const check of checks) {
-    try {
-      check.fn();
-      results.passed++;
-      results.requirements[check.id] = { status: 'pass', desc: check.desc };
-      console.log(`  PASS  ${check.id}: ${check.desc}`);
-    } catch (err) {
-      results.failed++;
-      results.requirements[check.id] = {
-        status: 'fail',
-        desc: check.desc,
-        error: err.message,
-      };
-      console.error(`  FAIL  ${check.id}: ${err.message}`);
-    }
-  }
-
-  // Cleanup tmp dir
-  try {
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  } catch {}
-  delete process.env.THUMBGATE_FEEDBACK_DIR;
-
-  // Write proof artifacts
-  fs.mkdirSync(proofDir, { recursive: true });
-
-  const report = {
-    phase: '07-data-quality',
-    generatedAt: new Date().toISOString(),
-    passed: results.passed,
-    failed: results.failed,
-    total: checks.length,
-    requirements: results.requirements,
-  };
-
-  fs.writeFileSync(reportJson, JSON.stringify(report, null, 2) + '\n');
-
-  const md = [
-    '# Phase 7: Data Quality — Proof Report',
-    '',
-    `Generated: ${report.generatedAt}`,
-    `Result: ${results.passed}/${checks.length} passed`,
-    '',
-    '## Requirements',
-    '',
-    ...Object.entries(results.requirements).map(([id, r]) => {
-      const checkbox = r.status === 'pass' ? '[x]' : '[ ]';
-      const errLine = r.error ? `\n  - Error: \`${r.error}\`` : '';
-      return `- ${checkbox} **${id}**: ${r.desc}${errLine}`;
-    }),
-    '',
-    '## Evidence',
-    '',
-    '- `scripts/validate-feedback.js` — 4-level validation pipeline (schema, semantics, anomaly, self-correction)',
-    '- `scripts/feedback-loop.js` — `inferOutcome()` and `enrichFeedbackContext()` added; `richContext` in every feedbackEvent',
-    '- `tests/validate-feedback.test.js` — 25 node:test cases covering all QUAL requirements',
-    '',
-  ].join('\n');
-
-  fs.writeFileSync(reportMd, md);
-
-  console.log(`\nPhase 7 proof: ${results.passed} passed, ${results.failed} failed`);
-  console.log(`Report: ${reportJson}`);
-
-  if (results.failed > 0) process.exit(1);
-}
-
-run();

package/scripts/prove-evolution.js

@@ -1,352 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const os = require('node:os');
-const path = require('node:path');
-
-const ROOT = path.join(__dirname, '..');
-const RUNNER_PATH = require.resolve('./async-job-runner');
-const FEEDBACK_PATH = require.resolve('./feedback-loop');
-const VERIFICATION_PATH = require.resolve('./verification-loop');
-const EXPERIMENT_TRACKER_PATH = require.resolve('./experiment-tracker');
-const EVOLUTION_STATE_PATH = require.resolve('./evolution-state');
-const WORKSPACE_EVOLVER_PATH = require.resolve('./workspace-evolver');
-const AUTORESEARCH_PATH = require.resolve('./autoresearch-runner');
-const VERIFY_RUN_PATH = require.resolve('./verify-run');
-
-function resolveProofPaths() {
-  const proofDir = process.env.THUMBGATE_PROOF_DIR || path.join(ROOT, 'proof');
-  return {
-    proofDir,
-    reportJson: path.join(proofDir, 'evolution-report.json'),
-    reportMd: path.join(proofDir, 'evolution-report.md'),
-  };
-}
-
-function resetModules() {
-  [
-    RUNNER_PATH,
-    FEEDBACK_PATH,
-    VERIFICATION_PATH,
-    EXPERIMENT_TRACKER_PATH,
-    EVOLUTION_STATE_PATH,
-    WORKSPACE_EVOLVER_PATH,
-    AUTORESEARCH_PATH,
-    VERIFY_RUN_PATH,
-  ].forEach((modulePath) => {
-    delete require.cache[modulePath];
-  });
-}
-
-function stubModule(modulePath, exports) {
-  require.cache[modulePath] = {
-    id: modulePath,
-    filename: modulePath,
-    loaded: true,
-    exports,
-  };
-}
-
-function buildStateCommand(settingKey, expectedValue) {
-  const script = [
-    'const { readEvolutionState } = require("./scripts/evolution-state");',
-    `const expected = ${JSON.stringify(expectedValue)};`,
-    `const value = readEvolutionState().settings[${JSON.stringify(settingKey)}];`,
-    'const passed = value === expected;',
-    'console.log("ℹ tests 1");',
-    'console.log("ℹ pass " + (passed ? 1 : 0));',
-    'console.log("ℹ fail " + (passed ? 0 : 1));',
-    'if (!passed) process.exit(1);',
-  ].join(' ');
-  return `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`;
-}
-
-function makeRejectedVerification() {
-  return {
-    accepted: false,
-    attempts: 2,
-    finalVerification: {
-      score: 0.2,
-      violations: [
-        {
-          pattern: 'webhook signature mismatch',
-          avoidRule: 'Verify webhook signatures before deploy.',
-        },
-      ],
-    },
-    partnerStrategy: {
-      profile: 'strict_reviewer',
-      verificationMode: 'evidence_first',
-    },
-    partnerReward: {
-      reward: 0,
-    },
-  };
-}
-
-async function run() {
-  const results = { passed: 0, failed: 0, requirements: {} };
-  const { proofDir, reportJson, reportMd } = resolveProofPaths();
-
-  const checks = [
-    {
-      id: 'EVOLVE-01',
-      desc: 'evolution-state loads defaults and captures rollback snapshots',
-      fn: () => {
-        const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-'));
-        try {
-          process.env.THUMBGATE_FEEDBACK_DIR = feedbackDir;
-          resetModules();
-          const state = require('./evolution-state');
-          const initial = state.readEvolutionState();
-          if (initial.settings.half_life_days !== 7) throw new Error('Expected default half_life_days=7');
-
-          const accepted = state.applyAcceptedMutation({
-            targetKey: 'half_life_days',
-            nextValue: 9,
-            experimentId: 'exp_proof',
-            summary: 'proof mutation',
-          });
-          if (!accepted.rollbackSnapshot.snapshotId) throw new Error('Expected rollback snapshot id');
-          if (state.readEvolutionState().settings.half_life_days !== 9) throw new Error('Accepted mutation did not persist');
-        } finally {
-          delete process.env.THUMBGATE_FEEDBACK_DIR;
-          resetModules();
-          fs.rmSync(feedbackDir, { recursive: true, force: true });
-        }
-      },
-    },
-    {
-      id: 'EVOLVE-02',
-      desc: 'workspace-evolver accepts improved candidates only when primary and holdout checks pass',
-      fn: () => {
-        const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-'));
-        try {
-          process.env.THUMBGATE_FEEDBACK_DIR = feedbackDir;
-          resetModules();
-          const { runWorkspaceEvolution } = require('./workspace-evolver');
-          const { readEvolutionState } = require('./evolution-state');
-          const result = runWorkspaceEvolution({
-            cwd: ROOT,
-            targetName: 'half_life_days',
-            nextValue: 8,
-            primaryCommands: [buildStateCommand('half_life_days', 8)],
-            holdoutCommands: [buildStateCommand('half_life_days', 8)],
-            timeoutMs: 5000,
-          });
-
-          if (!result.kept) throw new Error('Expected improved candidate to be kept');
-          if (!result.metrics.rollbackSnapshotId) throw new Error('Expected rollback snapshot metadata');
-          if (readEvolutionState().settings.half_life_days !== 8) throw new Error('Accepted evolution state not applied');
-        } finally {
-          delete process.env.THUMBGATE_FEEDBACK_DIR;
-          resetModules();
-          fs.rmSync(feedbackDir, { recursive: true, force: true });
-        }
-      },
-    },
-    {
-      id: 'EVOLVE-03',
-      desc: 'restoreWorkspaceEvolution rolls accepted state back to the previous snapshot',
-      fn: () => {
-        const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-'));
-        try {
-          process.env.THUMBGATE_FEEDBACK_DIR = feedbackDir;
-          resetModules();
-          const { runWorkspaceEvolution, restoreWorkspaceEvolution } = require('./workspace-evolver');
-          const { readEvolutionState } = require('./evolution-state');
-          const result = runWorkspaceEvolution({
-            cwd: ROOT,
-            targetName: 'half_life_days',
-            nextValue: 8,
-            primaryCommands: [buildStateCommand('half_life_days', 8)],
-            holdoutCommands: [buildStateCommand('half_life_days', 8)],
-            timeoutMs: 5000,
-          });
-
-          restoreWorkspaceEvolution(result.metrics.rollbackSnapshotId);
-          if (readEvolutionState().settings.half_life_days !== 7) throw new Error('Rollback did not restore default state');
-        } finally {
-          delete process.env.THUMBGATE_FEEDBACK_DIR;
-          resetModules();
-          fs.rmSync(feedbackDir, { recursive: true, force: true });
-        }
-      },
-    },
-    {
-      id: 'EVOLVE-04',
-      desc: 'autoresearch-runner delegates to workspace evolver and records research metadata plus rollback evidence',
-      fn: async () => {
-        const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-'));
-        try {
-          process.env.THUMBGATE_FEEDBACK_DIR = feedbackDir;
-          resetModules();
-          const runner = require('./autoresearch-runner');
-          const result = await runner.runIteration({
-            targetName: 'half_life_days',
-            nextValue: 8,
-            testCommand: buildStateCommand('half_life_days', 8),
-            holdoutCommands: [buildStateCommand('half_life_days', 8)],
-            timeoutMs: 5000,
-            researchQuery: 'rank fusion',
-            searchPapersImpl: async () => [{
-              paperId: '2603.01896',
-              title: 'Agentic Rank Fusion for Research Systems',
-              summary: 'Retrieval fusion for agent workflows.',
-              authors: ['Ada Lovelace'],
-              tags: ['retrieval'],
-              url: 'https://arxiv.org/abs/2603.01896',
-              source: 'huggingface-papers',
-            }],
-          });
-
-          if (!result.kept) throw new Error('Expected autoresearch iteration to keep the improved candidate');
-          if (result.metrics.researchQuery !== 'rank fusion') throw new Error('Research query metadata missing');
-          if (!result.metrics.researchPackId) throw new Error('Research pack id missing');
-          if (!result.metrics.researchPaperIds.includes('2603.01896')) throw new Error('Research paper id missing');
-          if (!result.metrics.rollbackSnapshotId) throw new Error('Rollback snapshot metadata missing');
-        } finally {
-          delete process.env.THUMBGATE_FEEDBACK_DIR;
-          resetModules();
-          fs.rmSync(feedbackDir, { recursive: true, force: true });
-        }
-      },
-    },
-    {
-      id: 'EVOLVE-05',
-      desc: 'async-job-runner follow-up experiments include a recommended evolution target and replay command',
-      fn: () => {
-        const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-'));
-        try {
-          process.env.THUMBGATE_FEEDBACK_DIR = feedbackDir;
-          resetModules();
-          stubModule(VERIFICATION_PATH, {
-            runVerificationLoop: () => makeRejectedVerification(),
-          });
-          const runner = require('./async-job-runner');
-          const tracker = require('./experiment-tracker');
-          runner.executeJob({
-            id: 'verification-failure-job',
-            context: 'webhook signature mismatch',
-            tags: ['billing'],
-            skill: 'billing-guard',
-          });
-          const experiments = tracker.loadExperiments();
-          const experiment = experiments[0];
-
-          if (!experiment) throw new Error('Expected queued improvement experiment');
-          if (!experiment.mutation.recommendedTarget) throw new Error('Missing recommendedTarget');
-          if (!experiment.mutation.evolutionCommand.includes('workspace-evolver.js')) {
-            throw new Error('Missing workspace evolver replay command');
-          }
-        } finally {
-          delete process.env.THUMBGATE_FEEDBACK_DIR;
-          resetModules();
-          fs.rmSync(feedbackDir, { recursive: true, force: true });
-        }
-      },
-    },
-    {
-      id: 'EVOLVE-06',
-      desc: 'verify:full includes the evolution proof lane and records the artifact',
-      fn: () => {
-        resetModules();
-        const { buildVerifyPlan, recordVerifyWorkflowRun } = require('./verify-run');
-        const plan = buildVerifyPlan('full');
-        const commands = plan.map((step) => [step.command, ...(step.args || [])].join(' ')).join('\n');
-        if (!commands.includes('prove:evolution')) {
-          throw new Error('verify:full is missing prove:evolution');
-        }
-
-        const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-feedback-'));
-        const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-evolution-proof-cwd-'));
-        try {
-          const entry = recordVerifyWorkflowRun('full', cwd, feedbackDir);
-          if (!entry.proofArtifacts.some((artifact) => artifact.endsWith(path.join('proof', 'evolution-report.json')))) {
-            throw new Error('Workflow run missing evolution proof artifact');
-          }
-        } finally {
-          fs.rmSync(feedbackDir, { recursive: true, force: true });
-          fs.rmSync(cwd, { recursive: true, force: true });
-        }
-      },
-    },
-  ];
-
-  console.log('Phase 17: Agent Workspace Evolution — Proof Gate\n');
-  console.log('Checking requirements:\n');
-
-  for (const check of checks) {
-    try {
-      await check.fn();
-      results.passed++;
-      results.requirements[check.id] = { status: 'pass', desc: check.desc };
-      console.log(`  PASS  ${check.id}: ${check.desc}`);
-    } catch (error) {
-      results.failed++;
-      results.requirements[check.id] = {
-        status: 'fail',
-        desc: check.desc,
-        error: error.message,
-      };
-      console.error(`  FAIL  ${check.id}: ${error.message}`);
-    }
-  }
-
-  fs.mkdirSync(proofDir, { recursive: true });
-
-  const report = {
-    phase: '17-agent-workspace-evolution',
-    generatedAt: new Date().toISOString(),
-    passed: results.passed,
-    failed: results.failed,
-    requirements: results.requirements,
-  };
-
-  fs.writeFileSync(reportJson, `${JSON.stringify(report, null, 2)}\n`);
-
-  const lines = [
-    '# Agent Workspace Evolution Proof Report',
-    '',
-    `Generated: ${report.generatedAt}`,
-    '',
-    `Summary: ${results.passed} passed, ${results.failed} failed`,
-    '',
-    '## Requirements',
-    '',
-  ];
-
-  for (const [id, requirement] of Object.entries(results.requirements)) {
-    lines.push(`- [${requirement.status === 'pass' ? 'x' : ' '}] **${id}** — ${requirement.desc}`);
-    if (requirement.error) {
-      lines.push(`  - Error: ${requirement.error}`);
-    }
-  }
-
-  lines.push('');
-  lines.push('## Evidence');
-  lines.push('');
-  lines.push('- `scripts/evolution-state.js` — local accepted-state overlay + rollback snapshots');
-  lines.push('- `scripts/workspace-evolver.js` — evolve / evaluate / accept / rollback engine');
-  lines.push('- `scripts/autoresearch-runner.js` — shared research-backed mutation loop on top of the evolver');
-  lines.push('- `scripts/prove-evolution.js` — this proof gate');
-
-  fs.writeFileSync(reportMd, `${lines.join('\n')}\n`);
-
-  console.log(`\n${results.passed} passed, ${results.failed} failed`);
-
-  if (results.failed > 0) {
-    process.exitCode = 1;
-  }
-}
-
-if (require.main === module) {
-  run().catch((error) => {
-    console.error(error);
-    process.exit(1);
-  });
-}
-
-module.exports = {
-  run,
-};