thumbgate 1.4.3 → 1.4.5

package/scripts/prove-harnesses.js

@@ -1,287 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-
-const ROOT = path.join(__dirname, '..');
-const HARNESS_PATH = require.resolve('./natural-language-harness');
-const RUNNER_PATH = require.resolve('./async-job-runner');
-const VERIFICATION_PATH = require.resolve('./verification-loop');
-const VERIFY_RUN_PATH = require.resolve('./verify-run');
-const SERVER_STDIO_PATH = require.resolve('../adapters/mcp/server-stdio');
-
-function resolveProofPaths() {
-  const proofDir = process.env.THUMBGATE_HARNESSES_PROOF_DIR || process.env.THUMBGATE_PROOF_DIR || path.join(ROOT, 'proof');
-  return {
-    proofDir,
-    reportJson: path.join(proofDir, 'harnesses-report.json'),
-    reportMd: path.join(proofDir, 'harnesses-report.md'),
-  };
-}
-
-function resetModules() {
-  [
-    HARNESS_PATH,
-    RUNNER_PATH,
-    VERIFICATION_PATH,
-    VERIFY_RUN_PATH,
-    SERVER_STDIO_PATH,
-  ].forEach((modulePath) => {
-    delete require.cache[modulePath];
-  });
-}
-
-function stubModule(modulePath, exports) {
-  require.cache[modulePath] = {
-    id: modulePath,
-    filename: modulePath,
-    loaded: true,
-    exports,
-  };
-}
-
-function makeAcceptedVerification() {
-  return {
-    accepted: true,
-    attempts: 1,
-    finalVerification: {
-      score: 1,
-      violations: [],
-    },
-    partnerStrategy: {
-      profile: 'strict_reviewer',
-      verificationMode: 'evidence_first',
-    },
-    partnerReward: {
-      reward: 1,
-    },
-  };
-}
-
-async function withHarnessRuntime(callback) {
-  const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-harness-proof-'));
-  process.env.THUMBGATE_FEEDBACK_DIR = feedbackDir;
-  try {
-    resetModules();
-    stubModule(VERIFICATION_PATH, {
-      runVerificationLoop: () => makeAcceptedVerification(),
-    });
-    return await callback({
-      harnesses: require('./natural-language-harness'),
-      runner: require('./async-job-runner'),
-      verifyRun: require('./verify-run'),
-      server: require('../adapters/mcp/server-stdio'),
-    });
-  } finally {
-    resetModules();
-    delete process.env.THUMBGATE_FEEDBACK_DIR;
-    fs.rmSync(feedbackDir, { recursive: true, force: true });
-  }
-}
-
-function writeReports(results, reportJson, reportMd) {
-  fs.mkdirSync(path.dirname(reportJson), { recursive: true });
-  fs.writeFileSync(reportJson, JSON.stringify(results, null, 2));
-
-  const lines = [
-    '# Natural-Language Harness Proof',
-    '',
-    `- Phase: ${results.phase}`,
-    `- Timestamp: ${results.timestamp}`,
-    `- ${results.passed} passed, ${results.failed} failed`,
-    '',
-    '## Requirements',
-    '',
-  ];
-
-  for (const requirement of Object.values(results.requirements)) {
-    lines.push(`- [${requirement.passed ? 'x' : ' '}] **${requirement.id}** ${requirement.desc}`);
-  }
-
-  fs.writeFileSync(reportMd, `${lines.join('\n')}\n`);
-}
-
-async function run() {
-  const results = {
-    phase: '23-natural-language-harnesses',
-    timestamp: new Date().toISOString(),
-    passed: 0,
-    failed: 0,
-    requirements: {},
-  };
-  const { reportJson, reportMd } = resolveProofPaths();
-
-  const checks = [
-    {
-      id: 'HARNESS-01',
-      desc: 'natural-language harness specs load and validate with required sections',
-      fn: async () => {
-        const { harnesses } = await withHarnessRuntime((context) => context);
-        const listed = harnesses.listHarnesses();
-        if (listed.length < 3) {
-          throw new Error(`Expected at least 3 harnesses, found ${listed.length}`);
-        }
-        if (!listed.some((entry) => entry.id === 'repo-full-verification')) {
-          throw new Error('repo-full-verification harness is missing');
-        }
-      },
-    },
-    {
-      id: 'HARNESS-02',
-      desc: 'rendered harness plans substitute inputs into natural-language steps',
-      fn: async () => {
-        await withHarnessRuntime(({ harnesses }) => {
-          const plan = harnesses.renderHarnessPlan('creator-partnership-review', {
-            creatorHandle: 'agentbuilder',
-          });
-          if (!plan.steps.some((step) => step.includes('agentbuilder'))) {
-            throw new Error('Expected rendered steps to include the input override');
-          }
-          if (!plan.successEvidence.some((line) => line.includes('agentbuilder'))) {
-            throw new Error('Expected rendered evidence to include the input override');
-          }
-        });
-      },
-    },
-    {
-      id: 'HARNESS-03',
-      desc: 'harness plans compile into executable async-job-runner stages',
-      fn: async () => {
-        await withHarnessRuntime(({ harnesses }) => {
-          const job = harnesses.buildHarnessJob('repo-full-verification', {
-            verificationCommand: 'node -e "process.stdout.write(\'verify ok\')"',
-          }, { jobId: 'proof-harness-job' });
-          const commandStage = job.stages.find((stage) => stage.command);
-          if (!commandStage || !commandStage.command.includes('verify ok')) {
-            throw new Error('Expected a command stage with the rendered verification command');
-          }
-          if (!job.tags.includes('natural-language-harness')) {
-            throw new Error('Expected natural-language-harness tag on compiled job');
-          }
-        });
-      },
-    },
-    {
-      id: 'HARNESS-04',
-      desc: 'runHarness executes a harness through the runtime with checkpoints and verification',
-      fn: async () => {
-        await withHarnessRuntime(({ harnesses, runner }) => {
-          const result = harnesses.runHarness('repo-full-verification', {
-            verificationCommand: 'node -e "process.stdout.write(\'verify ok\')"',
-          }, {
-            jobId: 'proof-run-harness',
-          });
-          const state = runner.readJobState('proof-run-harness');
-          if (result.status !== 'completed') {
-            throw new Error(`Expected completed harness result, got ${result.status}`);
-          }
-          if (!state || state.stageHistory.length < 3) {
-            throw new Error('Expected persisted stage history for executed harness');
-          }
-          if (!String(state.currentContext || '').includes('Success evidence required:')) {
-            throw new Error('Expected final context to include success evidence summary');
-          }
-        });
-      },
-    },
-    {
-      id: 'HARNESS-05',
-      desc: 'MCP surfaces expose list_harnesses and run_harness operations',
-      fn: () => {
-        return withHarnessRuntime(async ({ server }) => {
-          const listed = await server.handleRequest({
-            jsonrpc: '2.0',
-            id: 1,
-            method: 'tools/call',
-            params: {
-              name: 'list_harnesses',
-              arguments: { tag: 'verification' },
-            },
-          });
-          const catalog = JSON.parse(listed.content[0].text);
-          if (!catalog.harnesses.some((entry) => entry.id === 'repo-full-verification')) {
-            throw new Error('Expected repo-full-verification in MCP harness catalog');
-          }
-
-          const executed = await server.handleRequest({
-            jsonrpc: '2.0',
-            id: 2,
-            method: 'tools/call',
-            params: {
-              name: 'run_harness',
-              arguments: {
-                harness: 'repo-full-verification',
-                jobId: 'mcp-proof-harness',
-                inputs: {
-                  verificationCommand: 'node -e "process.stdout.write(\'verify ok\')"',
-                },
-              },
-            },
-          });
-          const payload = JSON.parse(executed.content[0].text);
-          if (payload.status !== 'completed') {
-            throw new Error(`Expected run_harness MCP result to complete, got ${payload.status}`);
-          }
-        });
-      },
-    },
-    {
-      id: 'HARNESS-06',
-      desc: 'full verification includes the harness proof lane and artifact',
-      fn: async () => {
-        await withHarnessRuntime(({ verifyRun }) => {
-          const plan = verifyRun.buildVerifyPlan('full');
-          const feedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-harness-run-'));
-          try {
-            const workflowRun = verifyRun.recordVerifyWorkflowRun('full', ROOT, feedbackDir);
-            if (!plan.some((step) => Array.isArray(step.args) && step.args.includes('prove:harnesses'))) {
-              throw new Error('verify:full is missing prove:harnesses');
-            }
-            if (!workflowRun.proofArtifacts.some((artifact) => artifact.endsWith(path.join('proof', 'harnesses-report.json')))) {
-              throw new Error('Workflow run is missing harness proof artifact');
-            }
-          } finally {
-            fs.rmSync(feedbackDir, { recursive: true, force: true });
-          }
-        });
-      },
-    },
-  ];
-
-  for (const check of checks) {
-    try {
-      await check.fn();
-      results.passed += 1;
-      results.requirements[check.id] = { id: check.id, desc: check.desc, passed: true };
-    } catch (error) {
-      results.failed += 1;
-      results.requirements[check.id] = {
-        id: check.id,
-        desc: check.desc,
-        passed: false,
-        error: error.message,
-      };
-    }
-  }
-
-  writeReports(results, reportJson, reportMd);
-
-  console.log(`${results.passed} passed, ${results.failed} failed`);
-  if (results.failed > 0) {
-    process.exitCode = 1;
-  }
-}
-
-if (require.main === module) {
-  run().catch((error) => {
-    console.error(error.stack || error.message);
-    process.exit(1);
-  });
-}
-
-module.exports = {
-  resolveProofPaths,
-  run,
-};

package/scripts/prove-intelligence.js

@@ -1,257 +0,0 @@
-#!/usr/bin/env node
-/**
- * prove-intelligence.js
- *
- * Smoke-test gate for Phase 9: Intelligence
- * Verifies context-engine and skill-quality-tracker work end-to-end.
- * Writes machine-readable JSON + human-readable markdown to proof/.
- */
-
-'use strict';
-
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const { execSync } = require('child_process');
-const { ensureDir } = require('./fs-utils');
-
-const ROOT = path.join(__dirname, '..');
-function getProofDir() {
-  return process.env.THUMBGATE_PROOF_DIR || path.join(ROOT, 'proof');
-}
-
-
-// ---------------------------------------------------------------------------
-// Run test suite and parse results
-// ---------------------------------------------------------------------------
-function runTests() {
-  try {
-    const output = execSync('node --test tests/intelligence.test.js', {
-      cwd: ROOT,
-      encoding: 'utf-8',
-      stdio: ['pipe', 'pipe', 'pipe'],
-    });
-    return output;
-  } catch (err) {
-    return err.stdout || err.stderr || String(err);
-  }
-}
-
-function parseTestOutput(output) {
-  const passMatch = output.match(/ℹ pass (\d+)/);
-  const failMatch = output.match(/ℹ fail (\d+)/);
-  const passed = passMatch ? parseInt(passMatch[1], 10) : 0;
-  const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
-  return { passed, failed };
-}
-
-// ---------------------------------------------------------------------------
-// Smoke test: context-engine
-// ---------------------------------------------------------------------------
-function smokeContextEngine() {
-  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-ce-'));
-  try {
-    delete require.cache[require.resolve('./context-engine.js')];
-    const ce = require('./context-engine.js');
-
-    // Build index from empty docs dir
-    const docsDir = path.join(tmpDir, 'docs');
-    fs.mkdirSync(docsDir, { recursive: true });
-    fs.writeFileSync(path.join(docsDir, 'CI_GUIDE.md'), '# CI Guide\nBuild pipeline guide.');
-    fs.writeFileSync(path.join(docsDir, 'MCP_SERVER.md'), '# MCP Server\nClaude MCP agent setup.');
-
-    const indexPath = path.join(tmpDir, 'idx.json');
-    const index = ce.buildKnowledgeIndex(docsDir, indexPath);
-
-    if (!index.bundles || !index.metadata) throw new Error('buildKnowledgeIndex missing bundles/metadata');
-    if (index.metadata.docCount !== 2) throw new Error(`Expected 2 docs, got ${index.metadata.docCount}`);
-
-    // Route query
-    // Query using keyword that will match ('guide' is extracted from title "CI Guide")
-    const result = ce.routeQuery('guide for pipeline', indexPath, 3);
-    if (!result.results || result.results.length === 0) throw new Error('routeQuery returned no results');
-
-    const cats = result.results.map((r) => r.category);
-    if (!cats.includes('ci-cd')) throw new Error(`ci-cd not in results: ${JSON.stringify(cats)}`);
-
-    // Prompt registry
-    const regPath = path.join(tmpDir, 'reg.json');
-    ce.registerPrompt('test-prompt', 'Hello {{name}}', { models: ['claude-opus-4-6'], category: 'test' }, regPath);
-    const prompt = ce.getPrompt('test-prompt', 'claude-opus-4-6', regPath);
-    if (!prompt || !prompt.compatible) throw new Error('registerPrompt/getPrompt failed');
-
-    return { passed: true, docsIndexed: 2, routingWorked: true, promptRegistry: true };
-  } catch (err) {
-    return { passed: false, error: err.message };
-  } finally {
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Smoke test: skill-quality-tracker
-// ---------------------------------------------------------------------------
-function smokeSkillTracker() {
-  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-sqt-'));
-  try {
-    delete require.cache[require.resolve('./skill-quality-tracker.js')];
-    const sqt = require('./skill-quality-tracker.js');
-
-    const now = Date.now();
-
-    // Write metrics
-    const metricsPath = path.join(tmpDir, 'metrics.jsonl');
-    const metrics = [
-      { tool_name: 'Read', timestamp: new Date(now).toISOString() },
-      { tool_name: 'Write', timestamp: new Date(now + 1000).toISOString() },
-      { tool_name: 'Read', timestamp: new Date(now + 2000).toISOString() },
-    ];
-    fs.writeFileSync(metricsPath, metrics.map((m) => JSON.stringify(m)).join('\n'));
-
-    // Write feedback (within window)
-    const feedbackPath = path.join(tmpDir, 'feedback.jsonl');
-    const feedback = [
-      { timestamp: new Date(now + 5000).toISOString(), feedback: 'up' },
-      { timestamp: new Date(now + 6000).toISOString(), signal: 'negative' },
-    ];
-    fs.writeFileSync(feedbackPath, feedback.map((f) => JSON.stringify(f)).join('\n'));
-
-    // Override env so processMetrics reads our test files
-    process.env.METRICS_PATH = metricsPath;
-    process.env.FEEDBACK_PATH = feedbackPath;
-
-    // Re-require after env change doesn't matter since we call functions directly
-    const breakdown = {
-      ConsistentSkill: { uses: 20, correlatedPositive: 18, correlatedNegative: 2 },
-      MixedSkill: { uses: 20, correlatedPositive: 10, correlatedNegative: 10 },
-    };
-    sqt.computeSuccessRates(breakdown);
-
-    if (!(breakdown.ConsistentSkill.successRate > breakdown.MixedSkill.successRate)) {
-      throw new Error('INTL-03: ConsistentSkill should score higher than MixedSkill');
-    }
-
-    const top = sqt.topPerformers(breakdown, 10, 5);
-    if (top.length === 0) throw new Error('topPerformers returned empty array');
-    if (top[0].tool !== 'ConsistentSkill') throw new Error('Expected ConsistentSkill as top performer');
-
-    const recs = sqt.generateRecommendations(top, [], breakdown);
-    if (!Array.isArray(recs) || recs.length === 0) throw new Error('generateRecommendations returned empty');
-
-    return {
-      passed: true,
-      correlationWindowMs: sqt.CORRELATION_WINDOW_MS,
-      consistentSuccessRate: breakdown.ConsistentSkill.successRate,
-      mixedSuccessRate: breakdown.MixedSkill.successRate,
-      intl03Satisfied: breakdown.ConsistentSkill.successRate > breakdown.MixedSkill.successRate,
-      topPerformer: top[0].tool,
-    };
-  } catch (err) {
-    return { passed: false, error: err.message };
-  } finally {
-    delete process.env.METRICS_PATH;
-    delete process.env.FEEDBACK_PATH;
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-async function main() {
-  console.log('Running Phase 9: Intelligence proof gate...\n');
-
-  const testOutput = runTests();
-  const { passed: testsPassed, failed: testsFailed } = parseTestOutput(testOutput);
-  const ceSmoke = smokeContextEngine();
-  const sqtSmoke = smokeSkillTracker();
-
-  const allPassed = testsFailed === 0 && ceSmoke.passed && sqtSmoke.passed;
-
-  const report = {
-    phase: 9,
-    name: 'Intelligence',
-    requirements: ['INTL-01', 'INTL-02', 'INTL-03'],
-    generatedAt: new Date().toISOString(),
-    testResults: {
-      passed: testsPassed,
-      failed: testsFailed,
-      suiteFile: 'tests/intelligence.test.js',
-    },
-    smokeTests: {
-      contextEngine: ceSmoke,
-      skillQualityTracker: sqtSmoke,
-    },
-    overallPassed: allPassed,
-  };
-
-  const proofDir = getProofDir();
-  ensureDir(proofDir);
-  const jsonPath = path.join(proofDir, 'intelligence-report.json');
-  const mdPath = path.join(proofDir, 'intelligence-report.md');
-
-  fs.writeFileSync(jsonPath, JSON.stringify(report, null, 2));
-
-  const status = allPassed ? 'PASSED' : 'FAILED';
-  const md = `# Phase 9: Intelligence — Proof Report
-
-**Status:** ${status}
-**Generated:** ${report.generatedAt}
-**Requirements:** ${report.requirements.join(', ')}
-
-## Test Results
-
-| Suite | Passed | Failed |
-|-------|--------|--------|
-| intelligence.test.js | ${testsPassed} | ${testsFailed} |
-
-## Smoke Tests
-
-### Context Engine (INTL-01)
-
-- Passed: ${ceSmoke.passed}
-${ceSmoke.passed ? `- Docs indexed: ${ceSmoke.docsIndexed}
-- Routing worked: ${ceSmoke.routingWorked}
-- Prompt registry: ${ceSmoke.promptRegistry}` : `- Error: ${ceSmoke.error}`}
-
-### Skill Quality Tracker (INTL-02, INTL-03)
-
-- Passed: ${sqtSmoke.passed}
-${sqtSmoke.passed ? `- Correlation window: ${sqtSmoke.correlationWindowMs}ms
-- Consistent skill success rate: ${sqtSmoke.consistentSuccessRate}
-- Mixed skill success rate: ${sqtSmoke.mixedSuccessRate}
-- INTL-03 satisfied (consistent > mixed): ${sqtSmoke.intl03Satisfied}
-- Top performer: ${sqtSmoke.topPerformer}` : `- Error: ${sqtSmoke.error}`}
-
-## Requirements Coverage
-
-| Requirement | Description | Status |
-|-------------|-------------|--------|
-| INTL-01 | Context engine routes queries to pre-computed bundles | ${ceSmoke.passed ? 'PASS' : 'FAIL'} |
-| INTL-02 | Skill tracker correlates tool calls to feedback by timestamp proximity | ${sqtSmoke.passed ? 'PASS' : 'FAIL'} |
-| INTL-03 | Both modules have unit tests (52 tests, 0 failures) | ${testsFailed === 0 ? 'PASS' : 'FAIL'} |
-
-## Files Created
-
-- \`scripts/context-engine.js\` — Knowledge bundle builder, context router, quality scorer, prompt registry
-- \`scripts/skill-quality-tracker.js\` — Tool call metric correlation to feedback by timestamp proximity
-- \`tests/intelligence.test.js\` — ${testsPassed} unit tests covering routing logic, correlation, edge cases
-- \`scripts/prove-intelligence.js\` — This proof gate script
-`;
-
-  fs.writeFileSync(mdPath, md);
-
-  console.log(`Status: ${status}`);
-  console.log(`Tests: ${testsPassed} passed, ${testsFailed} failed`);
-  console.log(`Context Engine smoke: ${ceSmoke.passed ? 'PASS' : 'FAIL'}`);
-  console.log(`Skill Tracker smoke: ${sqtSmoke.passed ? 'PASS' : 'FAIL'}`);
-  console.log(`\nReport written to: ${mdPath}`);
-  console.log(`JSON report: ${jsonPath}`);
-
-  process.exit(allPassed ? 0 : 1);
-}
-
-main().catch((err) => {
-  console.error('prove-intelligence failed:', err.message);
-  process.exit(1);
-});