rlhf-feedback-loop 0.5.0

package/scripts/prove-automation.js

@@ -0,0 +1,393 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const {
+  captureFeedback,
+  analyzeFeedback,
+  buildPreventionRules,
+  getFeedbackPaths,
+  readJSONL,
+} = require('./feedback-loop');
+const { exportDpoFromMemories } = require('./export-dpo-pairs');
+const { planIntent } = require('./intent-router');
+const { startServer } = require('../src/api/server');
+const { handleRequest } = require('../adapters/mcp/server-stdio');
+const { collectHealthReport } = require('./self-healing-check');
+const { runSelfHeal } = require('./self-heal');
+const { CONTEXTFS_ROOT, NAMESPACES } = require('./contextfs');
+const { traceForProofCheck, aggregateTraces } = require('./code-reasoning');
+
+const ROOT = path.join(__dirname, '..');
+const DEFAULT_PROOF_DIR = path.join(ROOT, 'proof', 'automation');
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function check(condition, message) {
+  if (!condition) throw new Error(message);
+}
+
+async function runAutomationProof(options = {}) {
+  const proofDir = options.proofDir || process.env.RLHF_AUTOMATION_PROOF_DIR || DEFAULT_PROOF_DIR;
+  const writeArtifacts = options.writeArtifacts !== false;
+  const proofPort = options.port ?? 0;
+
+  if (writeArtifacts) ensureDir(proofDir);
+
+  const tmpFeedbackDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rlhf-automation-proof-'));
+  process.env.RLHF_FEEDBACK_DIR = tmpFeedbackDir;
+  process.env.RLHF_API_KEY = 'automation-proof-key';
+  process.env.RLHF_MCP_PROFILE = 'default';
+
+  const report = {
+    generatedAt: new Date().toISOString(),
+    checks: [],
+    summary: { passed: 0, failed: 0 },
+  };
+
+  function addResult(name, passed, details) {
+    report.checks.push({ name, passed, details });
+    if (passed) report.summary.passed += 1;
+    else report.summary.failed += 1;
+  }
+
+  const { server, port } = await startServer({ port: proofPort });
+  try {
+    // 1) Positive with valid rubric -> accepted
+    {
+      const result = captureFeedback({
+        signal: 'up',
+        context: 'Implemented with tests and evidence',
+        whatWorked: 'Used proof harness and verification logs',
+        tags: ['verification', 'automation'],
+        rubricScores: [
+          { criterion: 'correctness', score: 4, evidence: 'all tests pass', judge: 'judge-a' },
+          { criterion: 'verification_evidence', score: 4, evidence: 'proof attached', judge: 'judge-a' },
+          { criterion: 'safety', score: 4, evidence: 'path checks enabled', judge: 'judge-a' },
+        ],
+        guardrails: {
+          testsPassed: true,
+          pathSafety: true,
+          budgetCompliant: true,
+        },
+      });
+      check(result.accepted === true, 'expected rubric-valid positive feedback to be accepted');
+      check(Boolean(result.memoryRecord && result.memoryRecord.rubricSummary), 'accepted learning should include rubricSummary');
+      addResult('feedback.capture.rubric_pass', true, {
+        accepted: result.accepted,
+        weightedScore: result.memoryRecord.rubricSummary.weightedScore,
+      });
+    }
+
+    // 2) Positive with failed guardrail/disagreement -> blocked
+    {
+      const result = captureFeedback({
+        signal: 'up',
+        context: 'Claimed done without logs',
+        whatWorked: 'looked good',
+        tags: ['verification', 'automation'],
+        rubricScores: [
+          { criterion: 'verification_evidence', score: 5, judge: 'judge-a' },
+          { criterion: 'verification_evidence', score: 2, judge: 'judge-b', evidence: 'logs missing' },
+        ],
+        guardrails: {
+          testsPassed: false,
+          pathSafety: true,
+          budgetCompliant: true,
+        },
+      });
+      check(result.accepted === false, 'expected rubric-gated positive feedback to be rejected');
+      check(/Rubric gate prevented promotion/i.test(String(result.reason)), 'expected rubric gate reason');
+      addResult('feedback.capture.rubric_block', true, { accepted: result.accepted, reason: result.reason });
+    }
+
+    // 3) Negative with rubric failures -> accepted mistake memory with rubric tags
+    {
+      const result = captureFeedback({
+        signal: 'down',
+        context: 'Skipped verification before completion claim',
+        whatWentWrong: 'No test evidence',
+        whatToChange: 'Always include test output',
+        tags: ['verification', 'automation'],
+        rubricScores: [
+          { criterion: 'verification_evidence', score: 1, evidence: 'no logs', judge: 'judge-a' },
+          { criterion: 'correctness', score: 2, evidence: 'regression detected', judge: 'judge-a' },
+        ],
+        guardrails: {
+          testsPassed: false,
+          pathSafety: true,
+          budgetCompliant: true,
+        },
+      });
+      check(result.accepted === true, 'expected negative feedback to be accepted as mistake memory');
+      check(result.memoryRecord.tags.includes('rubric-verification_evidence'), 'expected rubric failure tags');
+      addResult('feedback.capture.negative_with_rubric', true, {
+        accepted: result.accepted,
+        tags: result.memoryRecord.tags,
+      });
+    }
+
+    // 4) analytics tracks rubric blocks/failures
+    {
+      const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
+      const stats = analyzeFeedback(FEEDBACK_LOG_PATH);
+      check(stats.rubric.samples >= 3, 'expected rubric samples to be tracked');
+      check(stats.rubric.blockedPromotions >= 1, 'expected blocked rubric promotions to be tracked');
+      addResult('analytics.rubric_tracking', true, stats.rubric);
+    }
+
+    // 5) prevention rules include rubric dimensions
+    {
+      const markdown = buildPreventionRules(1);
+      check(markdown.includes('Rubric Failure Dimensions'), 'expected rubric section in prevention rules');
+      check(markdown.includes('verification_evidence'), 'expected criterion in prevention rules');
+      addResult('prevention_rules.rubric_dimensions', true, { hasRubricSection: true });
+    }
+
+    // 6) DPO export includes rubric delta metadata
+    {
+      const { MEMORY_LOG_PATH } = getFeedbackPaths();
+      const memories = readJSONL(MEMORY_LOG_PATH);
+      const result = exportDpoFromMemories(memories);
+      check(result.pairs.length >= 1, 'expected at least one DPO pair');
+      const first = result.pairs[0];
+      check(Boolean(first.metadata && first.metadata.rubric), 'expected rubric metadata in DPO pair');
+      addResult('dpo_export.rubric_metadata', true, first.metadata.rubric);
+    }
+
+    // 7) API rubric gate returns 422
+    {
+      const res = await fetch(`http://localhost:${port}/v1/feedback/capture`, {
+        method: 'POST',
+        headers: {
+          Authorization: 'Bearer automation-proof-key',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          signal: 'up',
+          context: 'unsafe api approval attempt',
+          whatWorked: 'claimed success',
+          tags: ['verification', 'automation'],
+          rubricScores: [
+            { criterion: 'verification_evidence', score: 5, judge: 'judge-a' },
+            { criterion: 'verification_evidence', score: 2, judge: 'judge-b', evidence: 'missing logs' },
+          ],
+          guardrails: { testsPassed: false, pathSafety: true, budgetCompliant: true },
+        }),
+      });
+      check(res.status === 422, `expected 422 from API rubric gate, got ${res.status}`);
+      const body = await res.json();
+      check(body.accepted === false, 'API rubric-gated capture must be rejected');
+      addResult('api.rubric_gate', true, { status: res.status });
+    }
+
+    // 8) MCP rubric gate returns accepted=false
+    {
+      const call = await handleRequest({
+        jsonrpc: '2.0',
+        id: 91,
+        method: 'tools/call',
+        params: {
+          name: 'capture_feedback',
+          arguments: {
+            signal: 'up',
+            context: 'unsafe mcp approval attempt',
+            whatWorked: 'claimed success',
+            rubricScores: [
+              { criterion: 'verification_evidence', score: 5, judge: 'judge-a' },
+              { criterion: 'verification_evidence', score: 2, judge: 'judge-b', evidence: 'missing logs' },
+            ],
+            guardrails: { testsPassed: false, pathSafety: true, budgetCompliant: true },
+          },
+        },
+      });
+      const payload = JSON.parse(call.content[0].text);
+      check(payload.accepted === false, 'MCP rubric-gated capture must be rejected');
+      addResult('mcp.rubric_gate', true, { accepted: payload.accepted });
+    }
+
+    // 9) intent checkpoints still enforced
+    {
+      const planBlocked = planIntent({
+        intentId: 'publish_dpo_training_data',
+        mcpProfile: 'default',
+        approved: false,
+      });
+      check(planBlocked.status === 'checkpoint_required', 'expected checkpoint_required for high-risk intent');
+
+      const planApproved = planIntent({
+        intentId: 'publish_dpo_training_data',
+        mcpProfile: 'default',
+        approved: true,
+      });
+      check(planApproved.status === 'ready', 'expected ready when approved');
+      addResult('intent.checkpoint_enforcement', true, {
+        blocked: planBlocked.status,
+        approved: planApproved.status,
+      });
+    }
+
+    // 10) context evaluate stores rubric evaluation
+    {
+      const construct = await fetch(`http://localhost:${port}/v1/context/construct`, {
+        method: 'POST',
+        headers: {
+          Authorization: 'Bearer automation-proof-key',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ query: 'verification automation', maxItems: 5, maxChars: 5000 }),
+      });
+      check(construct.status === 200, `context construct expected 200, got ${construct.status}`);
+      const pack = await construct.json();
+
+      const evaluate = await fetch(`http://localhost:${port}/v1/context/evaluate`, {
+        method: 'POST',
+        headers: {
+          Authorization: 'Bearer automation-proof-key',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          packId: pack.packId,
+          outcome: 'useful',
+          signal: 'positive',
+          rubricScores: [
+            { criterion: 'correctness', score: 4, evidence: 'tests pass', judge: 'judge-a' },
+            { criterion: 'verification_evidence', score: 4, evidence: 'logs attached', judge: 'judge-a' },
+          ],
+          guardrails: { testsPassed: true, pathSafety: true, budgetCompliant: true },
+        }),
+      });
+      check(evaluate.status === 200, `context evaluate expected 200, got ${evaluate.status}`);
+      const evalBody = await evaluate.json();
+      check(Boolean(evalBody.rubricEvaluation), 'expected rubricEvaluation on context evaluate result');
+      addResult('context.evaluate.rubric', true, { rubricId: evalBody.rubricEvaluation.rubricId });
+    }
+
+    // 11) semantic cache hit on equivalent query
+    {
+      fs.rmSync(path.join(CONTEXTFS_ROOT, NAMESPACES.provenance, 'semantic-cache.jsonl'), { force: true });
+      const first = await fetch(`http://localhost:${port}/v1/context/construct`, {
+        method: 'POST',
+        headers: {
+          Authorization: 'Bearer automation-proof-key',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ query: 'verification testing evidence', maxItems: 5, maxChars: 5000 }),
+      });
+      check(first.status === 200, `first context construct expected 200, got ${first.status}`);
+      const firstPack = await first.json();
+
+      const second = await fetch(`http://localhost:${port}/v1/context/construct`, {
+        method: 'POST',
+        headers: {
+          Authorization: 'Bearer automation-proof-key',
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ query: 'testing verification evidence', maxItems: 5, maxChars: 5000 }),
+      });
+      check(second.status === 200, `second context construct expected 200, got ${second.status}`);
+      const secondPack = await second.json();
+      check(firstPack.cache && firstPack.cache.hit === false, 'first pack expected cache miss');
+      check(secondPack.cache && secondPack.cache.hit === true, 'second pack expected cache hit');
+      addResult('context.semantic_cache.hit', true, {
+        firstHit: firstPack.cache.hit,
+        secondHit: secondPack.cache.hit,
+        similarity: secondPack.cache.similarity,
+      });
+    }
+
+    // 12) self-healing helpers produce healthy reports in baseline state
+    {
+      const health = collectHealthReport({
+        checks: [
+          { name: 'noop', command: ['node', '-e', 'process.exit(0)'] },
+        ],
+      });
+      check(health.overall_status === 'healthy', 'health report expected healthy for noop check');
+
+      const heal = runSelfHeal({ reason: 'automation-proof', cwd: ROOT });
+      check(heal.healthy === true, 'self-heal expected healthy execution');
+      check(Boolean(heal.reasoning), 'self-heal must include reasoning traces');
+      check(heal.traces.length === heal.plan.length, 'self-heal traces count must match plan length');
+      addResult('self_healing.helpers', true, {
+        healthStatus: health.overall_status,
+        changed: heal.changed,
+        reasoning: heal.reasoning,
+      });
+    }
+
+    // 13) code reasoning traces verify DPO pair quality
+    {
+      const { MEMORY_LOG_PATH } = getFeedbackPaths();
+      const memories = readJSONL(MEMORY_LOG_PATH);
+      const result = exportDpoFromMemories(memories);
+      if (result.pairs.length >= 1) {
+        const first = result.pairs[0];
+        check(Boolean(first.metadata.reasoningTrace), 'DPO pair must include reasoningTrace metadata');
+        check(typeof first.metadata.reasoningTrace.confidence === 'number', 'reasoningTrace must have confidence score');
+        check(typeof first.metadata.reasoningTrace.traceId === 'string', 'reasoningTrace must have traceId');
+        check(Boolean(result.reasoning), 'DPO export must include aggregate reasoning summary');
+        addResult('code_reasoning.dpo_traces', true, {
+          traceId: first.metadata.reasoningTrace.traceId,
+          confidence: first.metadata.reasoningTrace.confidence,
+          aggregateConfidence: result.reasoning.averageConfidence,
+        });
+      } else {
+        addResult('code_reasoning.dpo_traces', true, { skipped: true, reason: 'no DPO pairs to trace' });
+      }
+    }
+
+    // 14) code reasoning traces attached to proof checks
+    {
+      const proofTraces = report.checks.map((chk) => traceForProofCheck(chk));
+      const aggregate = aggregateTraces(proofTraces);
+      check(aggregate.totalTraces === report.checks.length, 'proof trace count must match check count');
+      check(aggregate.refuted === 0, 'no proof check should have refuted steps');
+      check(aggregate.averageConfidence > 0, 'proof traces must have positive confidence');
+      report.reasoning = aggregate;
+      report.proofTraces = proofTraces;
+      addResult('code_reasoning.proof_gate', true, {
+        totalTraces: aggregate.totalTraces,
+        averageConfidence: aggregate.averageConfidence,
+        allPassed: aggregate.allPassed,
+      });
+    }
+  } catch (err) {
+    addResult('fatal', false, { error: err.message });
+  } finally {
+    await new Promise((resolve) => server.close(resolve));
+    fs.rmSync(tmpFeedbackDir, { recursive: true, force: true });
+  }
+
+  if (writeArtifacts) {
+    fs.writeFileSync(path.join(proofDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`);
+    const mdLines = [
+      '# Automation Proof',
+      '',
+      `Generated: ${report.generatedAt}`,
+      '',
+      `Passed: ${report.summary.passed}`,
+      `Failed: ${report.summary.failed}`,
+      '',
+      '## Checks',
+      ...report.checks.map((checkItem) => `- ${checkItem.passed ? 'PASS' : 'FAIL'} ${checkItem.name}`),
+      '',
+    ];
+    fs.writeFileSync(path.join(proofDir, 'report.md'), `${mdLines.join('\n')}\n`);
+  }
+
+  if (report.summary.failed > 0) process.exitCode = 1;
+  return report;
+}
+
+module.exports = {
+  runAutomationProof,
+};
+
+if (require.main === module) {
+  runAutomationProof().then((report) => {
+    console.log(JSON.stringify(report.summary, null, 2));
+  });
+}

package/scripts/prove-data-quality.js

@@ -0,0 +1,219 @@
+'use strict';
+/**
+ * Phase 7: Data Quality — Proof Gate
+ *
+ * Validates all QUAL-01 through QUAL-04 requirements offline.
+ * Mirrors the pattern of prove-attribution.js (mkdtempSync + env override + execSync).
+ *
+ * Usage:
+ *   node scripts/prove-data-quality.js
+ *
+ * Produces:
+ *   proof/data-quality-report.json
+ *   proof/data-quality-report.md
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const PROOF_DIR = path.join(__dirname, '..', 'proof');
+const REPORT_JSON = path.join(PROOF_DIR, 'data-quality-report.json');
+const REPORT_MD = path.join(PROOF_DIR, 'data-quality-report.md');
+
+function run() {
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rlhf-qual-proof-'));
+  const results = { passed: 0, failed: 0, requirements: {} };
+
+  const checks = [
+    {
+      id: 'QUAL-01',
+      desc: 'validate-feedback.js exports validateEntry with 4-level pipeline',
+      fn: () => {
+        delete require.cache[require.resolve('./validate-feedback')];
+        const v = require('./validate-feedback');
+        if (typeof v.validateEntry !== 'function') throw new Error('validateEntry not exported');
+        if (typeof v.validateSchema !== 'function') throw new Error('validateSchema not exported');
+        if (typeof v.validateSemantics !== 'function') throw new Error('validateSemantics not exported');
+        if (typeof v.detectAnomalies !== 'function') throw new Error('detectAnomalies not exported');
+        if (typeof v.generateCorrections !== 'function') throw new Error('generateCorrections not exported');
+
+        // Verify semantic inconsistency is caught
+        const r = v.validateEntry({
+          id: 'proof-x',
+          timestamp: new Date().toISOString(),
+          signal: 'positive',
+          reward: -1,
+          context: 'good work done',
+        });
+        if (r.valid) throw new Error('Expected invalid for positive+negative-reward');
+        if (!r.corrections.length) throw new Error('Expected auto-correction for reward');
+        if (r.correctedEntry.reward !== 1) throw new Error('Expected corrected reward=1');
+
+        // Verify sensitive data detection
+        const r2 = v.validateEntry({
+          id: 'proof-y',
+          timestamp: new Date().toISOString(),
+          signal: 'positive',
+          reward: 1,
+          context: 'api_key=abc123 was in the response',
+        });
+        if (!r2.issues.some((i) => i.type === 'security')) {
+          throw new Error('Expected security issue for api_key pattern');
+        }
+      },
+    },
+    {
+      id: 'QUAL-02',
+      desc: 'captureFeedback produces richContext with domain, filePaths, errorType, outcomeCategory',
+      fn: () => {
+        process.env.RLHF_FEEDBACK_DIR = tmpDir;
+        // Clear module cache so env var takes effect
+        [
+          './feedback-loop',
+          './feedback-attribution',
+          './rlaif-self-audit',
+        ].forEach((m) => {
+          try {
+            delete require.cache[require.resolve(m)];
+          } catch {
+            // optional module
+          }
+        });
+        const { captureFeedback } = require('./feedback-loop');
+        const r = captureFeedback({
+          signal: 'positive',
+          context: 'unit tests added for edge cases',
+          tags: ['testing'],
+          filePaths: ['src/api.js'],
+        });
+        if (!r.feedbackEvent) throw new Error('No feedbackEvent in result');
+        const rc = r.feedbackEvent.richContext;
+        if (!rc) throw new Error('richContext missing from feedbackEvent');
+        if (typeof rc.domain !== 'string') throw new Error('richContext.domain must be string');
+        if (!Array.isArray(rc.filePaths)) throw new Error('richContext.filePaths must be array');
+        if (!('errorType' in rc)) throw new Error('richContext.errorType field missing');
+        if (typeof rc.outcomeCategory !== 'string') throw new Error('richContext.outcomeCategory must be string');
+        if (rc.domain !== 'testing') throw new Error(`Expected domain=testing, got ${rc.domain}`);
+      },
+    },
+    {
+      id: 'QUAL-03',
+      desc: 'inferOutcome returns granular categories beyond binary up/down',
+      fn: () => {
+        [
+          './feedback-loop',
+        ].forEach((m) => {
+          try { delete require.cache[require.resolve(m)]; } catch {}
+        });
+        const { inferOutcome } = require('./feedback-loop');
+        if (typeof inferOutcome !== 'function') throw new Error('inferOutcome not exported from feedback-loop');
+
+        const cases = [
+          ['positive', 'solved it first try', 'quick-success'],
+          ['positive', 'thorough comprehensive analysis', 'deep-success'],
+          ['positive', 'worked well overall', 'standard-success'],
+          ['negative', 'gave wrong incorrect answer', 'factual-error'],
+          ['negative', 'shallow surface level response', 'insufficient-depth'],
+          ['negative', 'guessed without checking docs', 'false-assumption'],
+        ];
+
+        for (const [signal, context, expected] of cases) {
+          const got = inferOutcome(signal, context);
+          if (got !== expected) {
+            throw new Error(`inferOutcome('${signal}', '${context}') = '${got}', expected '${expected}'`);
+          }
+        }
+      },
+    },
+    {
+      id: 'QUAL-04',
+      desc: 'test:quality (node --test tests/validate-feedback.test.js) passes with 0 failures',
+      fn: () => {
+        const out = execSync('node --test tests/validate-feedback.test.js', {
+          cwd: path.join(__dirname, '..'),
+          env: { ...process.env, RLHF_FEEDBACK_DIR: tmpDir },
+          encoding: 'utf8',
+          stdio: 'pipe',
+        });
+        // node:test exits non-zero on failure — if we get here, all tests passed
+        const failMatch = out.match(/ℹ fail (\d+)/);
+        if (failMatch && parseInt(failMatch[1], 10) > 0) {
+          throw new Error(`Tests failed: ${failMatch[1]} failure(s)\n${out.slice(-500)}`);
+        }
+      },
+    },
+  ];
+
+  console.log('Phase 7: Data Quality — Proof Gate\n');
+  console.log('Checking requirements:\n');
+
+  for (const check of checks) {
+    try {
+      check.fn();
+      results.passed++;
+      results.requirements[check.id] = { status: 'pass', desc: check.desc };
+      console.log(`  PASS  ${check.id}: ${check.desc}`);
+    } catch (err) {
+      results.failed++;
+      results.requirements[check.id] = {
+        status: 'fail',
+        desc: check.desc,
+        error: err.message,
+      };
+      console.error(`  FAIL  ${check.id}: ${err.message}`);
+    }
+  }
+
+  // Cleanup tmp dir
+  try {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  } catch {}
+  delete process.env.RLHF_FEEDBACK_DIR;
+
+  // Write proof artifacts
+  fs.mkdirSync(PROOF_DIR, { recursive: true });
+
+  const report = {
+    phase: '07-data-quality',
+    generatedAt: new Date().toISOString(),
+    passed: results.passed,
+    failed: results.failed,
+    total: checks.length,
+    requirements: results.requirements,
+  };
+
+  fs.writeFileSync(REPORT_JSON, JSON.stringify(report, null, 2) + '\n');
+
+  const md = [
+    '# Phase 7: Data Quality — Proof Report',
+    '',
+    `Generated: ${report.generatedAt}`,
+    `Result: ${results.passed}/${checks.length} passed`,
+    '',
+    '## Requirements',
+    '',
+    ...Object.entries(results.requirements).map(([id, r]) => {
+      const checkbox = r.status === 'pass' ? '[x]' : '[ ]';
+      const errLine = r.error ? `\n  - Error: \`${r.error}\`` : '';
+      return `- ${checkbox} **${id}**: ${r.desc}${errLine}`;
+    }),
+    '',
+    '## Evidence',
+    '',
+    '- `scripts/validate-feedback.js` — 4-level validation pipeline (schema, semantics, anomaly, self-correction)',
+    '- `scripts/feedback-loop.js` — `inferOutcome()` and `enrichFeedbackContext()` added; `richContext` in every feedbackEvent',
+    '- `tests/validate-feedback.test.js` — 25 node:test cases covering all QUAL requirements',
+    '',
+  ].join('\n');
+
+  fs.writeFileSync(REPORT_MD, md);
+
+  console.log(`\nPhase 7 proof: ${results.passed} passed, ${results.failed} failed`);
+  console.log(`Report: ${REPORT_JSON}`);
+
+  if (results.failed > 0) process.exit(1);
+}
+
+run();