rlhf-feedback-loop 0.5.0

package/scripts/prove-attribution.js

@@ -0,0 +1,364 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * prove-attribution.js — Phase 6 gate proof script.
+ *
+ * Generates proof/attribution-report.md and proof/attribution-report.json
+ * documenting per-requirement evidence for ATTR-01, ATTR-02, ATTR-03.
+ *
+ * Mirrors the prove-rlaif.js structure exactly (mkdtempSync / env override /
+ * execSync node --test / write JSON + markdown report).
+ *
+ * Exit 0 if no 'fail' statuses; exit 1 if any 'fail'.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { execSync } = require('child_process');
+
+const ROOT = path.join(__dirname, '..');
+const PROOF_DIR = path.join(ROOT, 'proof');
+
+// Phase 5 node-runner test baseline (before Phase 6 attribution tests)
+const PHASE5_BASELINE = 142;
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+async function runProof() {
+  const report = {
+    phase: '06-feedback-attribution',
+    generated: new Date().toISOString(),
+    requirements: {},
+    summary: { passed: 0, failed: 0 },
+  };
+
+  function addResult(reqId, reqStatus, evidence) {
+    report.requirements[reqId] = { status: reqStatus, evidence };
+    if (reqStatus === 'pass') report.summary.passed += 1;
+    else report.summary.failed += 1;
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // ATTR-01: recordAction + attributeFeedback correctness
+  // - require feedback-attribution.js with tmpDir env overrides
+  // - call recordAction('Bash', '{"command":"git push --force"}')
+  // - assert result.ok === true, result.action.intent === 'git-risk'
+  // - call attributeFeedback('negative', 'bad git push force broke main')
+  // - assert fs.existsSync(RLHF_FEEDBACK_ATTRIBUTIONS path)
+  // - parse JSONL, assert attribution_id and signal === 'negative'
+  // ─────────────────────────────────────────────────────────────────────────
+  const tmpDir01 = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-attr01-'));
+  let attr01Status = 'fail';
+  let attr01Evidence = '';
+  try {
+    process.env.RLHF_ACTION_LOG = path.join(tmpDir01, 'action-log.jsonl');
+    process.env.RLHF_FEEDBACK_ATTRIBUTIONS = path.join(tmpDir01, 'feedback-attributions.jsonl');
+    process.env.RLHF_ATTRIBUTED_FEEDBACK = path.join(tmpDir01, 'attributed-feedback.jsonl');
+
+    // Invalidate module cache so env vars take effect
+    for (const key of Object.keys(require.cache)) {
+      if (key.includes('feedback-attribution')) {
+        delete require.cache[key];
+      }
+    }
+    const { recordAction, attributeFeedback } = require('./feedback-attribution');
+
+    // Test recordAction
+    const recResult = recordAction('Bash', '{"command":"git push --force"}');
+    const recOk = recResult.ok === true && recResult.action.intent === 'git-risk';
+    const actionLogExists = fs.existsSync(path.join(tmpDir01, 'action-log.jsonl'));
+
+    // Test attributeFeedback — negative signal should write attributions
+    const attrResult = attributeFeedback('negative', 'bad git push force broke main');
+    const attrOk = attrResult.ok === true;
+    const attributionsPath = path.join(tmpDir01, 'feedback-attributions.jsonl');
+    const attributionsExist = fs.existsSync(attributionsPath);
+
+    let attributionValid = false;
+    if (attributionsExist) {
+      const lines = fs.readFileSync(attributionsPath, 'utf8').trim().split('\n').filter(Boolean);
+      if (lines.length > 0) {
+        try {
+          const parsed = JSON.parse(lines[lines.length - 1]);
+          attributionValid = typeof parsed.attribution_id === 'string' &&
+            parsed.attribution_id.startsWith('att_') &&
+            parsed.signal === 'negative';
+        } catch (_) {
+          attributionValid = false;
+        }
+      }
+    }
+
+    if (recOk && actionLogExists && attrOk && attributionsExist && attributionValid) {
+      attr01Status = 'pass';
+      attr01Evidence =
+        `recordAction('Bash', git push --force) returned ok=true, intent=git-risk. ` +
+        `action-log.jsonl written to ${tmpDir01}. ` +
+        `action_id=${recResult.action.action_id}, risk_score=${recResult.action.risk_score}. ` +
+        `attributeFeedback('negative', ...) returned ok=true, attributedCount=${attrResult.attributedCount}. ` +
+        `feedback-attributions.jsonl written. attribution_id=${attrResult.attributionId || 'written'}, signal=negative. ` +
+        `Module: scripts/feedback-attribution.js. Pure offline JSONL-based attribution.`;
+    } else {
+      const issues = [];
+      if (!recOk) issues.push(`recordAction returned ok=${recResult.ok}, intent=${recResult.action ? recResult.action.intent : 'none'} (expected git-risk)`);
+      if (!actionLogExists) issues.push(`action-log.jsonl not written to ${tmpDir01}`);
+      if (!attrOk) issues.push(`attributeFeedback returned ok=${attrResult.ok}`);
+      if (!attributionsExist) issues.push(`feedback-attributions.jsonl not written`);
+      if (!attributionValid) issues.push(`attribution entry missing attribution_id or signal`);
+      attr01Status = 'fail';
+      attr01Evidence = `ATTR-01 smoke test failed: ${issues.join('; ')}`;
+    }
+  } catch (err) {
+    attr01Status = 'fail';
+    attr01Evidence = `ATTR-01 threw: ${err.message}`;
+  } finally {
+    try { fs.rmSync(tmpDir01, { recursive: true, force: true }); } catch (_) {}
+    delete process.env.RLHF_ACTION_LOG;
+    delete process.env.RLHF_FEEDBACK_ATTRIBUTIONS;
+    delete process.env.RLHF_ATTRIBUTED_FEEDBACK;
+  }
+  addResult('ATTR-01', attr01Status, attr01Evidence);
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // ATTR-02: evaluatePretool allow/block/warn paths
+  // - require hybrid-feedback-context.js with tmpDir env overrides
+  // - seed attributed-feedback.jsonl with 3 negative entries for Bash + git push force
+  // - call buildHybridState() — should detect recurringNegativePatterns with count >= 3
+  // - call evaluatePretoolFromState(state, 'Bash', 'git push force main') → assert mode === 'block'
+  // - call evaluatePretoolFromState(state, 'Read', 'some-unrelated-file.md') → assert mode === 'allow'
+  // ─────────────────────────────────────────────────────────────────────────
+  const tmpDir02 = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-attr02-'));
+  let attr02Status = 'fail';
+  let attr02Evidence = '';
+  try {
+    process.env.RLHF_FEEDBACK_LOG = path.join(tmpDir02, 'feedback-log.jsonl');
+    process.env.RLHF_ATTRIBUTED_FEEDBACK = path.join(tmpDir02, 'attributed-feedback.jsonl');
+    process.env.RLHF_GUARDS_PATH = path.join(tmpDir02, 'pretool-guards.json');
+
+    // Seed attributed-feedback.jsonl with 3 identical negative entries
+    const attrFeedbackPath = path.join(tmpDir02, 'attributed-feedback.jsonl');
+    const ts = new Date().toISOString();
+    const seedEntries = [
+      {
+        timestamp: ts,
+        signal: 'negative',
+        feedback: 'negative',
+        tool_name: 'Bash',
+        context: 'git push force main branch override',
+        source: 'attributed',
+      },
+      {
+        timestamp: ts,
+        signal: 'negative',
+        feedback: 'negative',
+        tool_name: 'Bash',
+        context: 'git push force main branch override',
+        source: 'attributed',
+      },
+      {
+        timestamp: ts,
+        signal: 'negative',
+        feedback: 'negative',
+        tool_name: 'Bash',
+        context: 'git push force main branch override',
+        source: 'attributed',
+      },
+    ];
+    fs.mkdirSync(path.dirname(attrFeedbackPath), { recursive: true });
+    fs.writeFileSync(
+      attrFeedbackPath,
+      seedEntries.map((e) => JSON.stringify(e)).join('\n') + '\n',
+    );
+
+    // Invalidate module cache
+    for (const key of Object.keys(require.cache)) {
+      if (key.includes('hybrid-feedback-context')) {
+        delete require.cache[key];
+      }
+    }
+    const { buildHybridState, evaluatePretoolFromState } = require('./hybrid-feedback-context');
+
+    const state = buildHybridState({
+      feedbackLogPath: path.join(tmpDir02, 'feedback-log.jsonl'),
+      attributedFeedbackPath: attrFeedbackPath,
+    });
+
+    const hasRecurring = state.recurringNegativePatterns.length > 0;
+    const topCount = hasRecurring ? state.recurringNegativePatterns[0].count : 0;
+    const countOk = topCount >= 3;
+
+    // block path: git push force matches pattern
+    const blockResult = evaluatePretoolFromState(state, 'Bash', 'git push force main');
+    const blockOk = blockResult.mode === 'block';
+
+    // allow path: completely different tool+input
+    const allowResult = evaluatePretoolFromState(state, 'Read', 'some-unrelated-file.md');
+    const allowOk = allowResult.mode === 'allow';
+
+    if (hasRecurring && countOk && blockOk && allowOk) {
+      attr02Status = 'pass';
+      attr02Evidence =
+        `buildHybridState() detected ${state.recurringNegativePatterns.length} recurring pattern(s). ` +
+        `Top pattern count=${topCount} (>= 3 → critical). ` +
+        `evaluatePretoolFromState('Bash', 'git push force main') → mode=${blockResult.mode}. ` +
+        `evaluatePretoolFromState('Read', 'some-unrelated-file.md') → mode=${allowResult.mode}. ` +
+        `block + allow paths verified. No false positive for unrelated Read tool. ` +
+        `Module: scripts/hybrid-feedback-context.js. hasTwoKeywordHits enforces no-false-positive invariant.`;
+    } else {
+      const issues = [];
+      if (!hasRecurring) issues.push(`no recurring negative patterns detected (expected >= 1 from 3 identical entries)`);
+      if (!countOk) issues.push(`top pattern count=${topCount} (expected >= 3)`);
+      if (!blockOk) issues.push(`block path returned mode=${blockResult.mode} (expected block)`);
+      if (!allowOk) issues.push(`allow path returned mode=${allowResult.mode} (expected allow)`);
+      attr02Status = 'fail';
+      attr02Evidence = `ATTR-02 smoke test failed: ${issues.join('; ')}`;
+    }
+  } catch (err) {
+    attr02Status = 'fail';
+    attr02Evidence = `ATTR-02 threw: ${err.message}`;
+  } finally {
+    try { fs.rmSync(tmpDir02, { recursive: true, force: true }); } catch (_) {}
+    delete process.env.RLHF_FEEDBACK_LOG;
+    delete process.env.RLHF_ATTRIBUTED_FEEDBACK;
+    delete process.env.RLHF_GUARDS_PATH;
+  }
+  addResult('ATTR-02', attr02Status, attr02Evidence);
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // ATTR-03: node --test on both attribution test files exits 0
+  // execSync('node --test tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js')
+  // Parse stdout to count passing tests vs baseline
+  // status: 'pass' if exit code 0 and pass count >= 1
+  // ─────────────────────────────────────────────────────────────────────────
+  let attr03Status = 'fail';
+  let attr03Evidence = '';
+  let attrPassCount = 0;
+  let attrFailCount = 0;
+  try {
+    const testOutput = execSync(
+      'node --test tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js 2>&1',
+      { cwd: ROOT, timeout: 60000, encoding: 'utf-8' },
+    );
+
+    const passMatch = testOutput.match(/pass\s+(\d+)/);
+    const failMatch = testOutput.match(/fail\s+(\d+)/);
+    attrPassCount = passMatch ? parseInt(passMatch[1], 10) : 0;
+    attrFailCount = failMatch ? parseInt(failMatch[1], 10) : 0;
+
+    const meetsRequirement = attrPassCount >= 1 && attrFailCount === 0;
+
+    if (meetsRequirement) {
+      attr03Status = 'pass';
+      attr03Evidence =
+        `node --test (2 attribution test files): pass=${attrPassCount}, fail=${attrFailCount}. ` +
+        `Phase 5 baseline (test:api + test:proof + test:rlaif): ${PHASE5_BASELINE} tests. ` +
+        `Phase 6 adds ${attrPassCount} new attribution tests. ` +
+        `Total with attribution: ${PHASE5_BASELINE + attrPassCount} tests (node-runner only). ` +
+        `Files: tests/feedback-attribution.test.js (recordAction, attributeFeedback), ` +
+        `tests/hybrid-feedback-context.test.js (evaluatePretool, buildHybridState, compileGuardArtifact). ` +
+        `All tests use fs.mkdtempSync() tmpdir isolation — zero production feedback dirs touched.`;
+    } else {
+      attr03Status = 'fail';
+      attr03Evidence =
+        `node --test attribution files: pass=${attrPassCount}, fail=${attrFailCount}. ` +
+        `Expected >= 1 passing and 0 failures. ` +
+        `${attrFailCount > 0 ? `${attrFailCount} test(s) failing.` : `Only ${attrPassCount} tests passing (need >= 1).`}`;
+    }
+  } catch (err) {
+    const output = err.stdout || err.stderr || err.message || '';
+    const outStr = String(output);
+    const passMatch = outStr.match(/pass\s+(\d+)/);
+    const failMatch = outStr.match(/fail\s+(\d+)/);
+    attrPassCount = passMatch ? parseInt(passMatch[1], 10) : 0;
+    attrFailCount = failMatch ? parseInt(failMatch[1], 10) : 1;
+    attr03Status = 'fail';
+    attr03Evidence = `node --test attribution files exited non-zero (${attrFailCount} failures). Output: ${outStr.slice(0, 500)}`;
+  }
+  addResult('ATTR-03', attr03Status, attr03Evidence);
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // Write proof artifacts
+  // ─────────────────────────────────────────────────────────────────────────
+  ensureDir(PROOF_DIR);
+
+  const jsonPath = path.join(PROOF_DIR, 'attribution-report.json');
+  fs.writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`);
+
+  const mdLines = [
+    '# Feedback Attribution — Proof Report',
+    '',
+    `Generated: ${report.generated}`,
+    `Phase: ${report.phase}`,
+    '',
+    `**Passed: ${report.summary.passed} | Failed: ${report.summary.failed}**`,
+    '',
+    '## Requirements',
+    '',
+    '| Requirement | Status | Evidence |',
+    '|-------------|--------|----------|',
+    ...Object.entries(report.requirements).map(
+      ([reqId, { status: s, evidence }]) =>
+        `| ${reqId} | ${s.toUpperCase()} | ${evidence.replace(/\|/g, '\\|').replace(/\n/g, ' ')} |`
+    ),
+    '',
+    '## Requirement Details',
+    '',
+  ];
+
+  for (const [reqId, { status: s, evidence }] of Object.entries(report.requirements)) {
+    mdLines.push(`### ${reqId} — ${s.toUpperCase()}`);
+    mdLines.push('');
+    mdLines.push(evidence);
+    mdLines.push('');
+  }
+
+  mdLines.push('## Test Count Delta');
+  mdLines.push('');
+  mdLines.push('| Baseline (Phase 5 final) | Phase 6 Attribution Addition | Total (node-runner) |');
+  mdLines.push('|--------------------------|------------------------------|---------------------|');
+  mdLines.push(`| ${PHASE5_BASELINE} tests | +${attrPassCount} attribution tests (2 test files) | ${PHASE5_BASELINE + attrPassCount} |`);
+  mdLines.push('');
+  mdLines.push('Phase 6 (plan-03) added attribution test coverage:');
+  mdLines.push('- `tests/feedback-attribution.test.js` — recordAction(), attributeFeedback() (5 tests)');
+  mdLines.push('- `tests/hybrid-feedback-context.test.js` — evaluatePretool, buildHybridState, compileGuardArtifact (16 tests)');
+  mdLines.push('');
+  mdLines.push('All tests use `fs.mkdtempSync()` tmpdir isolation. Zero production feedback dirs touched.');
+  mdLines.push('');
+  mdLines.push('## Summary');
+  mdLines.push('');
+  mdLines.push(`${report.summary.passed}/3 requirements passed.`);
+  mdLines.push('');
+
+  const mdPath = path.join(PROOF_DIR, 'attribution-report.md');
+  fs.writeFileSync(mdPath, `${mdLines.join('\n')}\n`);
+
+  console.log(`Proof written to ${mdPath}`);
+  console.log(`           and   ${jsonPath}`);
+  console.log('');
+  console.log(JSON.stringify(report.summary, null, 2));
+
+  const hasFail = report.summary.failed > 0;
+  if (hasFail) {
+    process.exitCode = 1;
+    console.error('\nFAIL — one or more requirements did not pass. See proof/attribution-report.md for details.');
+  } else {
+    console.log('\nPASS — all requirements satisfied.');
+  }
+
+  return report;
+}
+
+module.exports = { runProof };
+
+if (require.main === module) {
+  runProof().catch((err) => {
+    console.error('Fatal error in prove-attribution.js:', err);
+    process.exitCode = 1;
+  });
+}