rlhf-feedback-loop 0.5.0

package/scripts/budget-guard.js

@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const FEEDBACK_DIR = process.env.RLHF_FEEDBACK_DIR || path.join(PROJECT_ROOT, '.claude', 'memory', 'feedback');
+const LEDGER_PATH = path.join(FEEDBACK_DIR, 'budget-ledger.json');
+const LOCK_PATH = `${LEDGER_PATH}.lock`;
+
+function parseMonthlyBudget(rawValue) {
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    throw new Error(`Invalid RLHF_MONTHLY_BUDGET_USD value: '${rawValue}'`);
+  }
+  return parsed;
+}
+
+function getMonthlyBudget() {
+  const rawValue = process.env.RLHF_MONTHLY_BUDGET_USD || '10';
+  return parseMonthlyBudget(rawValue);
+}
+
+function currentMonthKey() {
+  const now = new Date();
+  return `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`;
+}
+
+function loadLedger() {
+  if (!fs.existsSync(LEDGER_PATH)) return { months: {} };
+  return JSON.parse(fs.readFileSync(LEDGER_PATH, 'utf-8'));
+}
+
+function saveLedger(ledger) {
+  fs.mkdirSync(path.dirname(LEDGER_PATH), { recursive: true });
+  fs.writeFileSync(LEDGER_PATH, `${JSON.stringify(ledger, null, 2)}\n`);
+}
+
+function blockMs(ms) {
+  const start = Date.now();
+  while (Date.now() - start < ms) {
+    // Intentional synchronous short wait while lock clears.
+  }
+}
+
+function acquireLock({ timeoutMs = 5000, staleMs = 15000 } = {}) {
+  const startedAt = Date.now();
+  fs.mkdirSync(path.dirname(LOCK_PATH), { recursive: true });
+
+  while (true) {
+    try {
+      return fs.openSync(LOCK_PATH, 'wx');
+    } catch (err) {
+      if (err.code !== 'EEXIST') throw err;
+
+      try {
+        const stat = fs.statSync(LOCK_PATH);
+        if (Date.now() - stat.mtimeMs > staleMs) {
+          fs.rmSync(LOCK_PATH, { force: true });
+          continue;
+        }
+      } catch {
+        // lock disappeared between retries
+      }
+
+      if (Date.now() - startedAt > timeoutMs) {
+        throw new Error('Could not acquire budget ledger lock');
+      }
+      blockMs(20);
+    }
+  }
+}
+
+function releaseLock(lockFd) {
+  try {
+    fs.closeSync(lockFd);
+  } finally {
+    fs.rmSync(LOCK_PATH, { force: true });
+  }
+}
+
+function addSpend({ amountUsd, source, note }) {
+  if (!Number.isFinite(amountUsd) || amountUsd < 0) {
+    throw new Error('amountUsd must be a non-negative number');
+  }
+
+  const budgetUsd = getMonthlyBudget();
+  const lockFd = acquireLock();
+  try {
+    const ledger = loadLedger();
+    const month = currentMonthKey();
+    if (!ledger.months[month]) {
+      ledger.months[month] = {
+        totalUsd: 0,
+        entries: [],
+      };
+    }
+
+    const nextTotal = ledger.months[month].totalUsd + amountUsd;
+    if (nextTotal > budgetUsd) {
+      throw new Error(`Budget exceeded: ${nextTotal.toFixed(2)} > ${budgetUsd.toFixed(2)} USD/month`);
+    }
+
+    ledger.months[month].totalUsd = nextTotal;
+    ledger.months[month].entries.push({
+      ts: new Date().toISOString(),
+      source: source || 'unknown',
+      note: note || '',
+      amountUsd,
+    });
+
+    saveLedger(ledger);
+    return {
+      month,
+      totalUsd: ledger.months[month].totalUsd,
+      budgetUsd,
+    };
+  } finally {
+    releaseLock(lockFd);
+  }
+}
+
+function getBudgetStatus() {
+  const budgetUsd = getMonthlyBudget();
+  const ledger = loadLedger();
+  const month = currentMonthKey();
+  const total = ledger.months[month] ? ledger.months[month].totalUsd : 0;
+  return {
+    month,
+    totalUsd: total,
+    budgetUsd,
+    remainingUsd: Math.max(0, budgetUsd - total),
+  };
+}
+
+function runCli() {
+  const args = process.argv.slice(2);
+  if (args.includes('--status')) {
+    console.log(JSON.stringify(getBudgetStatus(), null, 2));
+    return;
+  }
+
+  const addArg = args.find((a) => a.startsWith('--add='));
+  if (!addArg) {
+    console.log('Usage: node scripts/budget-guard.js --status');
+    console.log('Usage: node scripts/budget-guard.js --add=0.15 --source=paperbanana --note="diagram generation"');
+    process.exit(1);
+  }
+
+  const amountUsd = Number(addArg.replace('--add=', ''));
+  const sourceArg = args.find((a) => a.startsWith('--source='));
+  const noteArg = args.find((a) => a.startsWith('--note='));
+
+  const result = addSpend({
+    amountUsd,
+    source: sourceArg ? sourceArg.replace('--source=', '') : 'unknown',
+    note: noteArg ? noteArg.replace('--note=', '') : '',
+  });
+
+  console.log(JSON.stringify(result, null, 2));
+}
+
+module.exports = {
+  addSpend,
+  getBudgetStatus,
+  getMonthlyBudget,
+  parseMonthlyBudget,
+  LEDGER_PATH,
+  LOCK_PATH,
+};
+
+if (require.main === module) {
+  runCli();
+}

package/scripts/code-reasoning.js

@@ -0,0 +1,307 @@
+#!/usr/bin/env node
+/**
+ * Agentic Code Reasoning — Structured Trace Engine
+ *
+ * Based on Meta's "Agentic Code Reasoning" paper (arxiv 2603.01896).
+ * Forces structured line-level reasoning instead of pattern-matching guesses.
+ *
+ * Produces a verification trace for every code change claim, self-heal fix,
+ * or DPO pair, requiring explicit evidence for each assertion.
+ */
+
+const crypto = require('node:crypto');
+
+/**
+ * @typedef {Object} TraceStep
+ * @property {string} location - File path and line range (e.g. "scripts/self-heal.js:49-69")
+ * @property {string} claim - What this step asserts about correctness
+ * @property {string} evidence - Concrete evidence supporting the claim
+ * @property {'verified'|'unverified'|'refuted'} verdict - Assessment of the claim
+ */
+
+/**
+ * @typedef {Object} ReasoningTrace
+ * @property {string} traceId - Unique identifier for this trace
+ * @property {string} timestamp - ISO 8601 timestamp
+ * @property {string} type - Trace type: 'self-heal' | 'dpo-pair' | 'proof-gate' | 'verification'
+ * @property {string} subject - What is being verified (script name, pair ID, etc.)
+ * @property {TraceStep[]} steps - Ordered reasoning steps
+ * @property {string[]} edgeCases - Edge cases explicitly addressed or ruled out
+ * @property {Object} summary - Aggregated verdict
+ * @property {number} summary.totalSteps - Total reasoning steps
+ * @property {number} summary.verified - Steps with verified verdict
+ * @property {number} summary.unverified - Steps with unverified verdict
+ * @property {number} summary.refuted - Steps with refuted verdict
+ * @property {number} summary.confidence - Ratio of verified to total (0-1)
+ * @property {boolean} summary.passed - True if confidence >= threshold and refuted === 0
+ */
+
+const DEFAULT_CONFIDENCE_THRESHOLD = 0.7;
+
+function generateTraceId() {
+  return `trace-${crypto.randomBytes(6).toString('hex')}`;
+}
+
+function createTrace(type, subject) {
+  return {
+    traceId: generateTraceId(),
+    timestamp: new Date().toISOString(),
+    type,
+    subject,
+    steps: [],
+    edgeCases: [],
+    summary: null,
+  };
+}
+
+function addStep(trace, { location, claim, evidence, verdict = 'unverified' }) {
+  if (!location || !claim) {
+    throw new Error('TraceStep requires location and claim');
+  }
+  const validVerdicts = ['verified', 'unverified', 'refuted'];
+  if (!validVerdicts.includes(verdict)) {
+    throw new Error(`Invalid verdict: ${verdict}. Must be one of: ${validVerdicts.join(', ')}`);
+  }
+  trace.steps.push({ location, claim, evidence: evidence || '', verdict });
+  return trace;
+}
+
+function addEdgeCase(trace, description) {
+  if (description) trace.edgeCases.push(description);
+  return trace;
+}
+
+function finalizeTrace(trace, { confidenceThreshold = DEFAULT_CONFIDENCE_THRESHOLD } = {}) {
+  const totalSteps = trace.steps.length;
+  const verified = trace.steps.filter((s) => s.verdict === 'verified').length;
+  const unverified = trace.steps.filter((s) => s.verdict === 'unverified').length;
+  const refuted = trace.steps.filter((s) => s.verdict === 'refuted').length;
+  const confidence = totalSteps > 0 ? Math.round((verified / totalSteps) * 1000) / 1000 : 0;
+
+  trace.summary = {
+    totalSteps,
+    verified,
+    unverified,
+    refuted,
+    confidence,
+    passed: confidence >= confidenceThreshold && refuted === 0,
+  };
+
+  return trace;
+}
+
+/**
+ * Build a reasoning trace for a self-heal fix execution.
+ *
+ * @param {Object} fixResult - Result from runFixPlan for a single script
+ * @param {string} fixResult.script - Script name
+ * @param {string} fixResult.status - 'success' | 'failed'
+ * @param {number} fixResult.exitCode
+ * @param {string} fixResult.outputTail - Last 2000 chars of output
+ * @param {string[]} changedFiles - Files changed by this fix
+ * @returns {ReasoningTrace}
+ */
+function traceForSelfHealFix(fixResult, changedFiles = []) {
+  const trace = createTrace('self-heal', fixResult.script);
+
+  addStep(trace, {
+    location: `npm run ${fixResult.script}`,
+    claim: `Fix script "${fixResult.script}" executes without error`,
+    evidence: fixResult.status === 'success'
+      ? `Exit code ${fixResult.exitCode}, completed in ${fixResult.durationMs}ms`
+      : `Exit code ${fixResult.exitCode}, error: ${fixResult.error || 'non-zero exit'}`,
+    verdict: fixResult.status === 'success' ? 'verified' : 'refuted',
+  });
+
+  if (changedFiles.length > 0) {
+    addStep(trace, {
+      location: changedFiles.join(', '),
+      claim: `Fix modified ${changedFiles.length} file(s) — changes are intentional`,
+      evidence: `Changed: ${changedFiles.join(', ')}`,
+      verdict: 'verified',
+    });
+  } else {
+    addStep(trace, {
+      location: `npm run ${fixResult.script}`,
+      claim: 'Fix produced no file changes (idempotent or no-op)',
+      evidence: 'git diff --name-only returned empty after execution',
+      verdict: 'verified',
+    });
+  }
+
+  const outputTail = (fixResult.outputTail || '').toLowerCase();
+  const hasErrors = /error|fail|exception|fatal/i.test(outputTail);
+  addStep(trace, {
+    location: `npm run ${fixResult.script} (output)`,
+    claim: 'Script output contains no error indicators',
+    evidence: hasErrors
+      ? `Output contains error keywords: ${outputTail.slice(-200)}`
+      : 'No error keywords in output tail',
+    verdict: hasErrors && fixResult.status === 'success' ? 'unverified' : (hasErrors ? 'refuted' : 'verified'),
+  });
+
+  addEdgeCase(trace, 'Script timeout not triggered (completed within deadline)');
+  if (changedFiles.length === 0) {
+    addEdgeCase(trace, 'No files changed — fix may already be applied or script is no-op');
+  }
+
+  return finalizeTrace(trace);
+}
+
+/**
+ * Build a reasoning trace for a DPO preference pair.
+ *
+ * @param {Object} pair - The DPO pair with prompt, chosen, rejected, metadata
+ * @returns {ReasoningTrace}
+ */
+function traceForDpoPair(pair) {
+  const trace = createTrace('dpo-pair', `${pair.metadata.errorId}->${pair.metadata.learningId}`);
+
+  addStep(trace, {
+    location: `error:${pair.metadata.errorId}`,
+    claim: 'Rejected response represents a genuine mistake',
+    evidence: `Error title: "${pair.metadata.errorTitle}"`,
+    verdict: pair.metadata.errorTitle ? 'verified' : 'unverified',
+  });
+
+  addStep(trace, {
+    location: `learning:${pair.metadata.learningId}`,
+    claim: 'Chosen response represents a correct approach',
+    evidence: `Learning title: "${pair.metadata.learningTitle}"`,
+    verdict: pair.metadata.learningTitle ? 'verified' : 'unverified',
+  });
+
+  const matchedKeys = pair.metadata.matchedKeys || [];
+  addStep(trace, {
+    location: 'domain-matching',
+    claim: `Error and learning share domain context (${matchedKeys.length} key(s))`,
+    evidence: `Matched keys: [${matchedKeys.join(', ')}], overlap score: ${pair.metadata.overlapScore}`,
+    verdict: matchedKeys.length > 0 ? 'verified' : 'refuted',
+  });
+
+  const rubric = pair.metadata.rubric;
+  if (rubric) {
+    const hasDelta = rubric.weightedDelta != null && rubric.weightedDelta > 0;
+    addStep(trace, {
+      location: 'rubric-delta',
+      claim: 'Learning scores higher than error on rubric (positive delta)',
+      evidence: `Learning: ${rubric.learningWeightedScore}, Error: ${rubric.errorWeightedScore}, Delta: ${rubric.weightedDelta}`,
+      verdict: hasDelta ? 'verified' : (rubric.weightedDelta === 0 ? 'unverified' : 'refuted'),
+    });
+
+    const failingCriteria = rubric.errorFailingCriteria || rubric.failingCriteria || [];
+    if (failingCriteria.length > 0) {
+      addStep(trace, {
+        location: 'rubric-failures',
+        claim: `Error had ${failingCriteria.length} failing rubric criteria`,
+        evidence: `Failing: [${failingCriteria.join(', ')}]`,
+        verdict: 'verified',
+      });
+    }
+  } else {
+    addStep(trace, {
+      location: 'rubric-delta',
+      claim: 'Rubric scores provide quantitative quality signal',
+      evidence: 'No rubric data available for this pair',
+      verdict: 'unverified',
+    });
+  }
+
+  addStep(trace, {
+    location: 'prompt-inference',
+    claim: 'Inferred prompt captures the shared scenario correctly',
+    evidence: `Prompt: "${pair.prompt}"`,
+    verdict: pair.prompt && pair.prompt.length > 10 ? 'verified' : 'unverified',
+  });
+
+  addEdgeCase(trace, 'Pair may lack temporal proximity — error and learning from different sessions');
+  addEdgeCase(trace, 'Domain overlap is keyword-based — semantic similarity not verified');
+
+  return finalizeTrace(trace);
+}
+
+/**
+ * Build a reasoning trace for a proof harness check.
+ *
+ * @param {Object} checkResult - A check from the proof report
+ * @param {string} checkResult.name - Check name
+ * @param {boolean} checkResult.passed - Whether the check passed
+ * @param {Object} checkResult.details - Check-specific details
+ * @returns {ReasoningTrace}
+ */
+function traceForProofCheck(checkResult) {
+  const trace = createTrace('proof-gate', checkResult.name);
+
+  addStep(trace, {
+    location: `check:${checkResult.name}`,
+    claim: `Proof check "${checkResult.name}" passes`,
+    evidence: checkResult.passed
+      ? `Passed with details: ${JSON.stringify(checkResult.details)}`
+      : `Failed: ${JSON.stringify(checkResult.details)}`,
+    verdict: checkResult.passed ? 'verified' : 'refuted',
+  });
+
+  if (checkResult.details) {
+    const details = checkResult.details;
+    if (details.status !== undefined) {
+      addStep(trace, {
+        location: `check:${checkResult.name}/status`,
+        claim: 'HTTP/response status is expected value',
+        evidence: `Status: ${details.status}`,
+        verdict: checkResult.passed ? 'verified' : 'refuted',
+      });
+    }
+    if (details.accepted !== undefined) {
+      addStep(trace, {
+        location: `check:${checkResult.name}/accepted`,
+        claim: `Acceptance state is ${details.accepted}`,
+        evidence: `accepted=${details.accepted}`,
+        verdict: 'verified',
+      });
+    }
+  }
+
+  return finalizeTrace(trace);
+}
+
+/**
+ * Aggregate multiple traces into a verification summary.
+ *
+ * @param {ReasoningTrace[]} traces
+ * @returns {Object} Aggregated summary
+ */
+function aggregateTraces(traces) {
+  const totalTraces = traces.length;
+  const passedTraces = traces.filter((t) => t.summary && t.summary.passed).length;
+  const allSteps = traces.flatMap((t) => t.steps);
+  const totalSteps = allSteps.length;
+  const verified = allSteps.filter((s) => s.verdict === 'verified').length;
+  const refuted = allSteps.filter((s) => s.verdict === 'refuted').length;
+  const avgConfidence = totalTraces > 0
+    ? Math.round(traces.reduce((sum, t) => sum + (t.summary ? t.summary.confidence : 0), 0) / totalTraces * 1000) / 1000
+    : 0;
+
+  return {
+    totalTraces,
+    passedTraces,
+    failedTraces: totalTraces - passedTraces,
+    totalSteps,
+    verified,
+    unverified: totalSteps - verified - refuted,
+    refuted,
+    averageConfidence: avgConfidence,
+    allPassed: passedTraces === totalTraces,
+  };
+}
+
+module.exports = {
+  createTrace,
+  addStep,
+  addEdgeCase,
+  finalizeTrace,
+  traceForSelfHealFix,
+  traceForDpoPair,
+  traceForProofCheck,
+  aggregateTraces,
+  DEFAULT_CONFIDENCE_THRESHOLD,
+};