thumbgate 1.4.3 → 1.4.5

package/scripts/prove-attribution.js

@@ -1,361 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-/**
- * prove-attribution.js — Phase 6 gate proof script.
- *
- * Generates proof/attribution-report.md and proof/attribution-report.json
- * documenting per-requirement evidence for ATTR-01, ATTR-02, ATTR-03.
- *
- * Mirrors the prove-rlaif.js structure exactly (mkdtempSync / env override /
- * execSync node --test / write JSON + markdown report).
- *
- * Exit 0 if no 'fail' statuses; exit 1 if any 'fail'.
- */
-
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const { execSync } = require('child_process');
-const { escapeMarkdownTableCell } = require('./markdown-escape');
-const { ensureDir } = require('./fs-utils');
-
-const ROOT = path.join(__dirname, '..');
-
-// Phase 5 node-runner test baseline (before Phase 6 attribution tests)
-const PHASE5_BASELINE = 142;
-
-
-async function runProof(options = {}) {
-  const proofDir = options.proofDir || process.env.THUMBGATE_PROOF_DIR || path.join(ROOT, 'proof');
-  const report = {
-    phase: '06-feedback-attribution',
-    generated: new Date().toISOString(),
-    requirements: {},
-    summary: { passed: 0, failed: 0 },
-  };
-
-  function addResult(reqId, reqStatus, evidence) {
-    report.requirements[reqId] = { status: reqStatus, evidence };
-    if (reqStatus === 'pass') report.summary.passed += 1;
-    else report.summary.failed += 1;
-  }
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // ATTR-01: recordAction + attributeFeedback correctness
-  // - require feedback-attribution.js with tmpDir env overrides
-  // - call recordAction('Bash', '{"command":"git push --force"}')
-  // - assert result.ok === true, result.action.intent === 'git-risk'
-  // - call attributeFeedback('negative', 'bad git push force broke main')
-  // - assert fs.existsSync(THUMBGATE_FEEDBACK_ATTRIBUTIONS path)
-  // - parse JSONL, assert attribution_id and signal === 'negative'
-  // ─────────────────────────────────────────────────────────────────────────
-  const tmpDir01 = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-attr01-'));
-  let attr01Status = 'fail';
-  let attr01Evidence = '';
-  try {
-    process.env.THUMBGATE_ACTION_LOG = path.join(tmpDir01, 'action-log.jsonl');
-    process.env.THUMBGATE_FEEDBACK_ATTRIBUTIONS = path.join(tmpDir01, 'feedback-attributions.jsonl');
-    process.env.THUMBGATE_ATTRIBUTED_FEEDBACK = path.join(tmpDir01, 'attributed-feedback.jsonl');
-
-    // Invalidate module cache so env vars take effect
-    for (const key of Object.keys(require.cache)) {
-      if (key.includes('feedback-attribution')) {
-        delete require.cache[key];
-      }
-    }
-    const { recordAction, attributeFeedback } = require('./feedback-attribution');
-
-    // Test recordAction
-    const recResult = recordAction('Bash', '{"command":"git push --force"}');
-    const recOk = recResult.ok === true && recResult.action.intent === 'git-risk';
-    const actionLogExists = fs.existsSync(path.join(tmpDir01, 'action-log.jsonl'));
-
-    // Test attributeFeedback — negative signal should write attributions
-    const attrResult = attributeFeedback('negative', 'bad git push force broke main');
-    const attrOk = attrResult.ok === true;
-    const attributionsPath = path.join(tmpDir01, 'feedback-attributions.jsonl');
-    const attributionsExist = fs.existsSync(attributionsPath);
-
-    let attributionValid = false;
-    if (attributionsExist) {
-      const lines = fs.readFileSync(attributionsPath, 'utf8').trim().split('\n').filter(Boolean);
-      if (lines.length > 0) {
-        try {
-          const parsed = JSON.parse(lines[lines.length - 1]);
-          attributionValid = typeof parsed.attribution_id === 'string' &&
-            parsed.attribution_id.startsWith('att_') &&
-            parsed.signal === 'negative';
-        } catch (_) {
-          attributionValid = false;
-        }
-      }
-    }
-
-    if (recOk && actionLogExists && attrOk && attributionsExist && attributionValid) {
-      attr01Status = 'pass';
-      attr01Evidence =
-        `recordAction('Bash', git push --force) returned ok=true, intent=git-risk. ` +
-        `action-log.jsonl written to ${tmpDir01}. ` +
-        `action_id=${recResult.action.action_id}, risk_score=${recResult.action.risk_score}. ` +
-        `attributeFeedback('negative', ...) returned ok=true, attributedCount=${attrResult.attributedCount}. ` +
-        `feedback-attributions.jsonl written. attribution_id=${attrResult.attributionId || 'written'}, signal=negative. ` +
-        `Module: scripts/feedback-attribution.js. Pure offline JSONL-based attribution.`;
-    } else {
-      const issues = [];
-      if (!recOk) issues.push(`recordAction returned ok=${recResult.ok}, intent=${recResult.action ? recResult.action.intent : 'none'} (expected git-risk)`);
-      if (!actionLogExists) issues.push(`action-log.jsonl not written to ${tmpDir01}`);
-      if (!attrOk) issues.push(`attributeFeedback returned ok=${attrResult.ok}`);
-      if (!attributionsExist) issues.push(`feedback-attributions.jsonl not written`);
-      if (!attributionValid) issues.push(`attribution entry missing attribution_id or signal`);
-      attr01Status = 'fail';
-      attr01Evidence = `ATTR-01 smoke test failed: ${issues.join('; ')}`;
-    }
-  } catch (err) {
-    attr01Status = 'fail';
-    attr01Evidence = `ATTR-01 threw: ${err.message}`;
-  } finally {
-    try { fs.rmSync(tmpDir01, { recursive: true, force: true }); } catch (_) {}
-    delete process.env.THUMBGATE_ACTION_LOG;
-    delete process.env.THUMBGATE_FEEDBACK_ATTRIBUTIONS;
-    delete process.env.THUMBGATE_ATTRIBUTED_FEEDBACK;
-  }
-  addResult('ATTR-01', attr01Status, attr01Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // ATTR-02: evaluatePretool allow/block/warn paths
-  // - require hybrid-feedback-context.js with tmpDir env overrides
-  // - seed attributed-feedback.jsonl with 3 negative entries for Bash + git push force
-  // - call buildHybridState() — should detect recurringNegativePatterns with count >= 3
-  // - call evaluatePretoolFromState(state, 'Bash', 'git push force main') → assert mode === 'block'
-  // - call evaluatePretoolFromState(state, 'Read', 'some-unrelated-file.md') → assert mode === 'allow'
-  // ─────────────────────────────────────────────────────────────────────────
-  const tmpDir02 = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-attr02-'));
-  let attr02Status = 'fail';
-  let attr02Evidence = '';
-  try {
-    process.env.THUMBGATE_FEEDBACK_LOG = path.join(tmpDir02, 'feedback-log.jsonl');
-    process.env.THUMBGATE_ATTRIBUTED_FEEDBACK = path.join(tmpDir02, 'attributed-feedback.jsonl');
-    process.env.THUMBGATE_GUARDS_PATH = path.join(tmpDir02, 'pretool-guards.json');
-
-    // Seed attributed-feedback.jsonl with 3 identical negative entries
-    const attrFeedbackPath = path.join(tmpDir02, 'attributed-feedback.jsonl');
-    const ts = new Date().toISOString();
-    const seedEntries = [
-      {
-        timestamp: ts,
-        signal: 'negative',
-        feedback: 'negative',
-        tool_name: 'Bash',
-        context: 'git push force main branch override',
-        source: 'attributed',
-      },
-      {
-        timestamp: ts,
-        signal: 'negative',
-        feedback: 'negative',
-        tool_name: 'Bash',
-        context: 'git push force main branch override',
-        source: 'attributed',
-      },
-      {
-        timestamp: ts,
-        signal: 'negative',
-        feedback: 'negative',
-        tool_name: 'Bash',
-        context: 'git push force main branch override',
-        source: 'attributed',
-      },
-    ];
-    fs.mkdirSync(path.dirname(attrFeedbackPath), { recursive: true });
-    fs.writeFileSync(
-      attrFeedbackPath,
-      seedEntries.map((e) => JSON.stringify(e)).join('\n') + '\n',
-    );
-
-    // Invalidate module cache
-    for (const key of Object.keys(require.cache)) {
-      if (key.includes('hybrid-feedback-context')) {
-        delete require.cache[key];
-      }
-    }
-    const { buildHybridState, evaluatePretoolFromState } = require('./hybrid-feedback-context');
-
-    const state = buildHybridState({
-      feedbackLogPath: path.join(tmpDir02, 'feedback-log.jsonl'),
-      attributedFeedbackPath: attrFeedbackPath,
-    });
-
-    const hasRecurring = state.recurringNegativePatterns.length > 0;
-    const topCount = hasRecurring ? state.recurringNegativePatterns[0].count : 0;
-    const countOk = topCount >= 3;
-
-    // block path: git push force matches pattern
-    const blockResult = evaluatePretoolFromState(state, 'Bash', 'git push force main');
-    const blockOk = blockResult.mode === 'block';
-
-    // allow path: completely different tool+input
-    const allowResult = evaluatePretoolFromState(state, 'Read', 'some-unrelated-file.md');
-    const allowOk = allowResult.mode === 'allow';
-
-    if (hasRecurring && countOk && blockOk && allowOk) {
-      attr02Status = 'pass';
-      attr02Evidence =
-        `buildHybridState() detected ${state.recurringNegativePatterns.length} recurring pattern(s). ` +
-        `Top pattern count=${topCount} (>= 3 → critical). ` +
-        `evaluatePretoolFromState('Bash', 'git push force main') → mode=${blockResult.mode}. ` +
-        `evaluatePretoolFromState('Read', 'some-unrelated-file.md') → mode=${allowResult.mode}. ` +
-        `block + allow paths verified. No false positive for unrelated Read tool. ` +
-        `Module: scripts/hybrid-feedback-context.js. hasTwoKeywordHits enforces no-false-positive invariant.`;
-    } else {
-      const issues = [];
-      if (!hasRecurring) issues.push(`no recurring negative patterns detected (expected >= 1 from 3 identical entries)`);
-      if (!countOk) issues.push(`top pattern count=${topCount} (expected >= 3)`);
-      if (!blockOk) issues.push(`block path returned mode=${blockResult.mode} (expected block)`);
-      if (!allowOk) issues.push(`allow path returned mode=${allowResult.mode} (expected allow)`);
-      attr02Status = 'fail';
-      attr02Evidence = `ATTR-02 smoke test failed: ${issues.join('; ')}`;
-    }
-  } catch (err) {
-    attr02Status = 'fail';
-    attr02Evidence = `ATTR-02 threw: ${err.message}`;
-  } finally {
-    try { fs.rmSync(tmpDir02, { recursive: true, force: true }); } catch (_) {}
-    delete process.env.THUMBGATE_FEEDBACK_LOG;
-    delete process.env.THUMBGATE_ATTRIBUTED_FEEDBACK;
-    delete process.env.THUMBGATE_GUARDS_PATH;
-  }
-  addResult('ATTR-02', attr02Status, attr02Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // ATTR-03: node --test on both attribution test files exits 0
-  // execSync('node --test tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js')
-  // Parse stdout to count passing tests vs baseline
-  // status: 'pass' if exit code 0 and pass count >= 1
-  // ─────────────────────────────────────────────────────────────────────────
-  let attr03Status = 'fail';
-  let attr03Evidence = '';
-  let attrPassCount = 0;
-  let attrFailCount = 0;
-  try {
-    const testOutput = execSync(
-      'node --test tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js 2>&1',
-      { cwd: ROOT, timeout: 60000, encoding: 'utf-8' },
-    );
-
-    const passMatch = testOutput.match(/pass\s+(\d+)/);
-    const failMatch = testOutput.match(/fail\s+(\d+)/);
-    attrPassCount = passMatch ? parseInt(passMatch[1], 10) : 0;
-    attrFailCount = failMatch ? parseInt(failMatch[1], 10) : 0;
-
-    const meetsRequirement = attrPassCount >= 1 && attrFailCount === 0;
-
-    if (meetsRequirement) {
-      attr03Status = 'pass';
-      attr03Evidence =
-        `node --test (2 attribution test files): pass=${attrPassCount}, fail=${attrFailCount}. ` +
-        `Phase 5 baseline (test:api + test:proof + test:rlaif): ${PHASE5_BASELINE} tests. ` +
-        `Phase 6 adds ${attrPassCount} new attribution tests. ` +
-        `Total with attribution: ${PHASE5_BASELINE + attrPassCount} tests (node-runner only). ` +
-        `Files: tests/feedback-attribution.test.js (recordAction, attributeFeedback), ` +
-        `tests/hybrid-feedback-context.test.js (evaluatePretool, buildHybridState, compileGuardArtifact). ` +
-        `All tests use fs.mkdtempSync() tmpdir isolation — zero production feedback dirs touched.`;
-    } else {
-      attr03Status = 'fail';
-      attr03Evidence =
-        `node --test attribution files: pass=${attrPassCount}, fail=${attrFailCount}. ` +
-        `Expected >= 1 passing and 0 failures. ` +
-        `${attrFailCount > 0 ? `${attrFailCount} test(s) failing.` : `Only ${attrPassCount} tests passing (need >= 1).`}`;
-    }
-  } catch (err) {
-    const output = err.stdout || err.stderr || err.message || '';
-    const outStr = String(output);
-    const passMatch = outStr.match(/pass\s+(\d+)/);
-    const failMatch = outStr.match(/fail\s+(\d+)/);
-    attrPassCount = passMatch ? parseInt(passMatch[1], 10) : 0;
-    attrFailCount = failMatch ? parseInt(failMatch[1], 10) : 1;
-    attr03Status = 'fail';
-    attr03Evidence = `node --test attribution files exited non-zero (${attrFailCount} failures). Output: ${outStr.slice(0, 500)}`;
-  }
-  addResult('ATTR-03', attr03Status, attr03Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // Write proof artifacts
-  // ─────────────────────────────────────────────────────────────────────────
-  ensureDir(proofDir);
-
-  const jsonPath = path.join(proofDir, 'attribution-report.json');
-  fs.writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`);
-
-  const mdLines = [
-    '# Feedback Attribution — Proof Report',
-    '',
-    `Generated: ${report.generated}`,
-    `Phase: ${report.phase}`,
-    '',
-    `**Passed: ${report.summary.passed} | Failed: ${report.summary.failed}**`,
-    '',
-    '## Requirements',
-    '',
-    '| Requirement | Status | Evidence |',
-    '|-------------|--------|----------|',
-    ...Object.entries(report.requirements).map(
-      ([reqId, { status: s, evidence }]) =>
-        `| ${reqId} | ${s.toUpperCase()} | ${escapeMarkdownTableCell(evidence)} |`
-    ),
-    '',
-    '## Requirement Details',
-    '',
-  ];
-
-  for (const [reqId, { status: s, evidence }] of Object.entries(report.requirements)) {
-    mdLines.push(`### ${reqId} — ${s.toUpperCase()}`);
-    mdLines.push('');
-    mdLines.push(evidence);
-    mdLines.push('');
-  }
-
-  mdLines.push('## Test Count Delta');
-  mdLines.push('');
-  mdLines.push('| Baseline (Phase 5 final) | Phase 6 Attribution Addition | Total (node-runner) |');
-  mdLines.push('|--------------------------|------------------------------|---------------------|');
-  mdLines.push(`| ${PHASE5_BASELINE} tests | +${attrPassCount} attribution tests (2 test files) | ${PHASE5_BASELINE + attrPassCount} |`);
-  mdLines.push('');
-  mdLines.push('Phase 6 (plan-03) added attribution test coverage:');
-  mdLines.push('- `tests/feedback-attribution.test.js` — recordAction(), attributeFeedback() (5 tests)');
-  mdLines.push('- `tests/hybrid-feedback-context.test.js` — evaluatePretool, buildHybridState, compileGuardArtifact (16 tests)');
-  mdLines.push('');
-  mdLines.push('All tests use `fs.mkdtempSync()` tmpdir isolation. Zero production feedback dirs touched.');
-  mdLines.push('');
-  mdLines.push('## Summary');
-  mdLines.push('');
-  mdLines.push(`${report.summary.passed}/3 requirements passed.`);
-  mdLines.push('');
-
-  const mdPath = path.join(proofDir, 'attribution-report.md');
-  fs.writeFileSync(mdPath, `${mdLines.join('\n')}\n`);
-
-  console.log(`Proof written to ${mdPath}`);
-  console.log(`           and   ${jsonPath}`);
-  console.log('');
-  console.log(JSON.stringify(report.summary, null, 2));
-
-  const hasFail = report.summary.failed > 0;
-  if (hasFail) {
-    process.exitCode = 1;
-    console.error(`\nFAIL — one or more requirements did not pass. See ${mdPath} for details.`);
-  } else {
-    console.log('\nPASS — all requirements satisfied.');
-  }
-
-  return report;
-}
-
-module.exports = { runProof };
-
-if (require.main === module) {
-  runProof().catch((err) => {
-    console.error('Fatal error in prove-attribution.js:', err);
-    process.exitCode = 1;
-  });
-}