rlhf-feedback-loop 0.5.0

package/scripts/rlaif-self-audit.js

@@ -0,0 +1,123 @@
+'use strict';
+/**
+ * RLAIF Self-Audit Module (DPO-01)
+ *
+ * Heuristic self-scoring of feedback events against CLAUDE.md constraints.
+ * NO API calls — pure synchronous evaluation of event fields.
+ *
+ * Exports: selfAudit, selfAuditAndLog, CONSTRAINTS
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+// ---------------------------------------------------------------------------
+// CLAUDE.md Constraint Definitions (weight sum = 1.0)
+// ---------------------------------------------------------------------------
+
+const CONSTRAINTS = [
+  {
+    id: 'has_context',
+    weight: 0.20,
+    check: (e) => typeof e.context === 'string' && e.context.length >= 20,
+  },
+  {
+    id: 'has_actionable_detail',
+    weight: 0.25,
+    check: (e) => {
+      if (e.signal === 'positive') return Boolean(e.whatWorked);
+      return Boolean(e.whatWentWrong) && Boolean(e.whatToChange);
+    },
+  },
+  {
+    id: 'schema_valid',
+    weight: 0.15,
+    check: (e) =>
+      ['positive', 'negative'].includes(e.signal) &&
+      Array.isArray(e.tags) &&
+      e.tags.length > 0,
+  },
+  {
+    id: 'rubric_evaluated',
+    weight: 0.20,
+    check: (e) => e.rubric != null && e.rubric.promotionEligible != null,
+  },
+  {
+    id: 'budget_compliant',
+    weight: 0.10,
+    check: (e) =>
+      !e.rubric ||
+      !e.rubric.failingGuardrails ||
+      !e.rubric.failingGuardrails.includes('budgetCompliant'),
+  },
+  {
+    id: 'no_vague_signal',
+    weight: 0.10,
+    check: (e) => typeof e.context === 'string' && e.context.length > 10,
+  },
+];
+
+// ---------------------------------------------------------------------------
+// selfAudit — pure function, no I/O, no API calls
+// ---------------------------------------------------------------------------
+
+/**
+ * Evaluate a feedback event against all CLAUDE.md constraints.
+ *
+ * @param {Object} feedbackEvent - A feedback event object
+ * @returns {{ score: number, constraints: Array, timestamp: string }}
+ *   score: float in [0, 1] rounded to 3 decimals
+ *   constraints: array of { constraint, passed, weight }
+ *   timestamp: ISO 8601 string at evaluation time
+ */
+function selfAudit(feedbackEvent) {
+  const e = feedbackEvent || {};
+  const results = CONSTRAINTS.map((c) => ({
+    constraint: c.id,
+    passed: Boolean(c.check(e)),
+    weight: c.weight,
+  }));
+  const score = results.reduce((sum, r) => sum + (r.passed ? r.weight : 0), 0);
+  return {
+    score: Math.round(score * 1000) / 1000,
+    constraints: results,
+    timestamp: new Date().toISOString(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// selfAuditAndLog — evaluates and appends to self-score-log.jsonl (sync)
+// ---------------------------------------------------------------------------
+
+/**
+ * Score a feedback event and append the result to self-score-log.jsonl.
+ *
+ * Non-critical: any filesystem error is swallowed — result is returned regardless.
+ *
+ * @param {Object} feedbackEvent - The feedback event to score
+ * @param {Object} mlPaths - Object with feedbackDir property (from getFeedbackPaths())
+ * @returns {{ score: number, constraints: Array, timestamp: string }}
+ */
+function selfAuditAndLog(feedbackEvent, mlPaths) {
+  const result = selfAudit(feedbackEvent);
+  try {
+    const feedbackDir = (mlPaths && mlPaths.FEEDBACK_DIR) || (mlPaths && mlPaths.feedbackDir);
+    if (feedbackDir) {
+      const logPath = path.join(feedbackDir, 'self-score-log.jsonl');
+      const entry = {
+        feedbackId: (feedbackEvent || {}).id || null,
+        ...result,
+      };
+      fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`);
+    }
+  } catch (_err) {
+    // Non-critical side-effect — swallow and return result anyway
+  }
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = { selfAudit, selfAuditAndLog, CONSTRAINTS };

package/scripts/rubric-engine.js

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_RUBRIC_PATH = path.join(PROJECT_ROOT, 'config', 'rubrics', 'default-v1.json');
+
+function getRubricPath() {
+  return process.env.RLHF_RUBRIC_PATH || DEFAULT_RUBRIC_PATH;
+}
+
+function loadRubricConfig() {
+  const raw = fs.readFileSync(getRubricPath(), 'utf-8');
+  const rubric = JSON.parse(raw);
+
+  if (!rubric || typeof rubric !== 'object') {
+    throw new Error('Invalid rubric config: expected object');
+  }
+  if (!rubric.rubricId || typeof rubric.rubricId !== 'string') {
+    throw new Error('Invalid rubric config: rubricId is required');
+  }
+  if (!Array.isArray(rubric.criteria) || rubric.criteria.length === 0) {
+    throw new Error('Invalid rubric config: criteria must be a non-empty array');
+  }
+
+  const seen = new Set();
+  let totalWeight = 0;
+  rubric.criteria.forEach((criterion) => {
+    if (!criterion.id || typeof criterion.id !== 'string') {
+      throw new Error('Invalid rubric config: criterion id is required');
+    }
+    if (seen.has(criterion.id)) {
+      throw new Error(`Invalid rubric config: duplicate criterion '${criterion.id}'`);
+    }
+    seen.add(criterion.id);
+
+    const weight = Number(criterion.weight);
+    if (!Number.isFinite(weight) || weight <= 0) {
+      throw new Error(`Invalid rubric config: criterion '${criterion.id}' has invalid weight`);
+    }
+    totalWeight += weight;
+
+    const minPassingScore = Number(criterion.minPassingScore || 3);
+    if (!Number.isFinite(minPassingScore) || minPassingScore < 1 || minPassingScore > 5) {
+      throw new Error(`Invalid rubric config: criterion '${criterion.id}' has invalid minPassingScore`);
+    }
+  });
+
+  if (Math.abs(totalWeight - 1) > 0.001) {
+    throw new Error(`Invalid rubric config: weights must sum to 1.0 (got ${totalWeight.toFixed(3)})`);
+  }
+
+  return rubric;
+}
+
+function parseRubricScores(input) {
+  if (input == null) return [];
+  if (Array.isArray(input)) return input;
+  if (typeof input === 'string') {
+    const trimmed = input.trim();
+    if (!trimmed) return [];
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) {
+      throw new Error('rubricScores must be an array');
+    }
+    return parsed;
+  }
+  throw new Error('rubricScores must be array or JSON string');
+}
+
+function normalizeRubricScores(rawScores, rubric = loadRubricConfig()) {
+  const scores = parseRubricScores(rawScores);
+  if (scores.length === 0) return [];
+
+  const criterionMap = new Map(rubric.criteria.map((c) => [c.id, c]));
+  return scores.map((item, index) => {
+    if (!item || typeof item !== 'object') {
+      throw new Error(`rubricScores[${index}] must be an object`);
+    }
+    const criterion = String(item.criterion || '').trim();
+    if (!criterionMap.has(criterion)) {
+      throw new Error(`rubricScores[${index}] unknown criterion '${criterion}'`);
+    }
+
+    const score = Number(item.score);
+    if (!Number.isFinite(score) || score < 1 || score > 5) {
+      throw new Error(`rubricScores[${index}] score must be between 1 and 5`);
+    }
+
+    return {
+      criterion,
+      score,
+      evidence: item.evidence ? String(item.evidence).trim() : '',
+      judge: item.judge ? String(item.judge).trim() : 'unknown',
+    };
+  });
+}
+
+function groupByCriterion(scores) {
+  const grouped = {};
+  scores.forEach((scoreItem) => {
+    if (!grouped[scoreItem.criterion]) grouped[scoreItem.criterion] = [];
+    grouped[scoreItem.criterion].push(scoreItem);
+  });
+  return grouped;
+}
+
+function evaluateGuardrails(guardrails, rubric = loadRubricConfig()) {
+  const input = guardrails && typeof guardrails === 'object' ? guardrails : {};
+  const expected = Array.isArray(rubric.guardrails) ? rubric.guardrails : [];
+
+  const status = {};
+  const failed = [];
+  expected.forEach((g) => {
+    const value = input[g.key];
+    const normalized = value === true ? true : value === false ? false : null;
+    status[g.key] = normalized;
+    if (normalized === false) failed.push(g.key);
+  });
+
+  return {
+    status,
+    failed,
+  };
+}
+
+function evaluateJudgeAgreement(scoresByCriterion) {
+  const disagreements = [];
+  for (const [criterion, entries] of Object.entries(scoresByCriterion)) {
+    if (entries.length < 2) continue;
+    const values = entries.map((e) => e.score);
+    const max = Math.max(...values);
+    const min = Math.min(...values);
+    if (max - min >= 2) {
+      disagreements.push({
+        criterion,
+        max,
+        min,
+        judges: entries.map((e) => e.judge),
+      });
+    }
+  }
+  return disagreements;
+}
+
+function buildRubricEvaluation({ rubricScores, guardrails } = {}) {
+  const rubric = loadRubricConfig();
+  const normalizedScores = normalizeRubricScores(rubricScores, rubric);
+  const scoresByCriterion = groupByCriterion(normalizedScores);
+  const guardrailResult = evaluateGuardrails(guardrails, rubric);
+
+  const criterionBreakdown = {};
+  const failingCriteria = [];
+  const missingEvidenceClaims = [];
+  let weightedScore = 0;
+
+  rubric.criteria.forEach((criterion) => {
+    const entries = scoresByCriterion[criterion.id] || [];
+    const avg = entries.length > 0
+      ? entries.reduce((sum, item) => sum + item.score, 0) / entries.length
+      : null;
+
+    criterionBreakdown[criterion.id] = {
+      averageScore: avg,
+      minPassingScore: Number(criterion.minPassingScore || 3),
+      judgeCount: entries.length,
+      label: criterion.label || criterion.id,
+    };
+
+    if (avg != null) {
+      weightedScore += (avg / 5) * Number(criterion.weight);
+      if (avg < Number(criterion.minPassingScore || 3)) {
+        failingCriteria.push(criterion.id);
+      }
+    }
+
+    if (criterion.requiresEvidence && entries.some((entry) => entry.score >= 4 && !entry.evidence)) {
+      missingEvidenceClaims.push(criterion.id);
+    }
+  });
+
+  const judgeDisagreements = evaluateJudgeAgreement(scoresByCriterion);
+  const blockReasons = [];
+  if (failingCriteria.length > 0) blockReasons.push(`failing_criteria:${failingCriteria.join(',')}`);
+  if (guardrailResult.failed.length > 0) blockReasons.push(`failed_guardrails:${guardrailResult.failed.join(',')}`);
+  if (judgeDisagreements.length > 0) blockReasons.push('judge_disagreement');
+  if (missingEvidenceClaims.length > 0) blockReasons.push(`missing_evidence:${missingEvidenceClaims.join(',')}`);
+
+  return {
+    rubricId: rubric.rubricId,
+    rubricVersion: rubric.version || 1,
+    weightedScore: Math.round(weightedScore * 1000) / 1000,
+    criterionBreakdown,
+    failingCriteria,
+    guardrails: guardrailResult.status,
+    failingGuardrails: guardrailResult.failed,
+    judgeDisagreements,
+    missingEvidenceClaims,
+    promotionEligible: blockReasons.length === 0,
+    blockReasons,
+    rubricScores: normalizedScores,
+  };
+}
+
+module.exports = {
+  DEFAULT_RUBRIC_PATH,
+  getRubricPath,
+  loadRubricConfig,
+  parseRubricScores,
+  normalizeRubricScores,
+  evaluateGuardrails,
+  evaluateJudgeAgreement,
+  buildRubricEvaluation,
+};
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const scoresArg = args.find((arg) => arg.startsWith('--scores='));
+  const guardrailsArg = args.find((arg) => arg.startsWith('--guardrails='));
+
+  if (!scoresArg) {
+    console.log(JSON.stringify(loadRubricConfig(), null, 2));
+    process.exit(0);
+  }
+
+  const scores = scoresArg.replace('--scores=', '');
+  const guardrails = guardrailsArg ? JSON.parse(guardrailsArg.replace('--guardrails=', '')) : {};
+  const result = buildRubricEvaluation({ rubricScores: scores, guardrails });
+  console.log(JSON.stringify(result, null, 2));
+}

package/scripts/self-heal.js

@@ -0,0 +1,127 @@
+#!/usr/bin/env node
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const { traceForSelfHealFix, aggregateTraces } = require('./code-reasoning');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const PACKAGE_JSON_PATH = path.join(PROJECT_ROOT, 'package.json');
+const KNOWN_FIX_SCRIPTS = ['lint:fix', 'format', 'fix', 'feedback:rules'];
+
+function runCommand(command, { cwd = PROJECT_ROOT, timeoutMs = 5 * 60_000 } = {}) {
+  const [cmd, ...args] = command;
+  const started = Date.now();
+  const result = spawnSync(cmd, args, {
+    cwd,
+    encoding: 'utf-8',
+    timeout: timeoutMs,
+    shell: false,
+  });
+
+  return {
+    command: command.join(' '),
+    exitCode: Number.isInteger(result.status) ? result.status : 1,
+    durationMs: Date.now() - started,
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    error: result.error ? result.error.message : null,
+  };
+}
+
+function loadPackageScripts(packageJsonPath = PACKAGE_JSON_PATH) {
+  const raw = fs.readFileSync(packageJsonPath, 'utf-8');
+  const parsed = JSON.parse(raw);
+  return parsed.scripts || {};
+}
+
+function buildFixPlan(scripts) {
+  return KNOWN_FIX_SCRIPTS.filter((name) => Object.prototype.hasOwnProperty.call(scripts, name));
+}
+
+function listChangedFiles({ cwd = PROJECT_ROOT } = {}) {
+  const diff = runCommand(['git', 'diff', '--name-only'], { cwd, timeoutMs: 10_000 });
+  if (diff.exitCode !== 0) return [];
+  return diff.stdout
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean);
+}
+
+function runFixPlan({ plan, runner = runCommand, cwd = PROJECT_ROOT } = {}) {
+  const results = [];
+  plan.forEach((scriptName) => {
+    const filesBefore = new Set(listChangedFiles({ cwd }));
+    const run = runner(['npm', 'run', scriptName], { cwd, timeoutMs: 10 * 60_000 });
+    const filesAfter = listChangedFiles({ cwd });
+    const scriptChangedFiles = filesAfter.filter((f) => !filesBefore.has(f));
+    results.push({
+      script: scriptName,
+      status: run.exitCode === 0 ? 'success' : 'failed',
+      exitCode: run.exitCode,
+      durationMs: run.durationMs,
+      error: run.error,
+      outputTail: `${run.stdout}\n${run.stderr}`.trim().slice(-2000),
+      changedFiles: scriptChangedFiles,
+    });
+  });
+
+  const successful = results.filter((x) => x.status === 'success').length;
+  return {
+    successful,
+    failed: results.length - successful,
+    total: results.length,
+    results,
+  };
+}
+
+function runSelfHeal({ reason = 'unknown', cwd = PROJECT_ROOT } = {}) {
+  const beforeChanges = listChangedFiles({ cwd });
+  const beforeSet = new Set(beforeChanges);
+  const scripts = loadPackageScripts();
+  const plan = buildFixPlan(scripts);
+  const execution = runFixPlan({ plan, cwd });
+  const afterChanges = listChangedFiles({ cwd });
+  const changedFiles = afterChanges.filter((filePath) => !beforeSet.has(filePath));
+
+  const traces = execution.results.map((fixResult) => {
+    return traceForSelfHealFix(fixResult, fixResult.changedFiles || []);
+  });
+  const reasoning = aggregateTraces(traces);
+
+  return {
+    timestamp: new Date().toISOString(),
+    reason,
+    plan,
+    execution,
+    preExistingChanges: beforeChanges,
+    changedFiles,
+    changed: changedFiles.length > 0,
+    healthy: execution.failed === 0,
+    reasoning,
+    traces,
+  };
+}
+
+function runCli() {
+  const reasonArg = process.argv.slice(2).find((a) => a.startsWith('--reason='));
+  const reason = reasonArg ? reasonArg.slice('--reason='.length) : 'manual';
+  const report = runSelfHeal({ reason });
+  console.log(JSON.stringify(report, null, 2));
+
+  if (!report.healthy) {
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  KNOWN_FIX_SCRIPTS,
+  loadPackageScripts,
+  buildFixPlan,
+  runFixPlan,
+  runSelfHeal,
+};
+
+if (require.main === module) {
+  runCli();
+}

package/scripts/self-healing-check.js

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+const { spawnSync } = require('node:child_process');
+const path = require('node:path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+
+const DEFAULT_CHECKS = [
+  { name: 'budget_status', command: ['npm', 'run', 'budget:status'], timeoutMs: 60_000 },
+  { name: 'tests', command: ['npm', 'test'], timeoutMs: 15 * 60_000 },
+  { name: 'prove_adapters', command: ['npm', 'run', 'prove:adapters'], timeoutMs: 10 * 60_000 },
+  { name: 'prove_automation', command: ['npm', 'run', 'prove:automation'], timeoutMs: 10 * 60_000 },
+];
+
+function runCommand(command, { cwd = PROJECT_ROOT, timeoutMs = 5 * 60_000 } = {}) {
+  const [cmd, ...args] = command;
+  const started = Date.now();
+  const result = spawnSync(cmd, args, {
+    cwd,
+    encoding: 'utf-8',
+    timeout: timeoutMs,
+    shell: false,
+  });
+
+  const durationMs = Date.now() - started;
+  const status = Number.isInteger(result.status) ? result.status : 1;
+  return {
+    exitCode: status,
+    durationMs,
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    error: result.error ? result.error.message : null,
+  };
+}
+
+function collectHealthReport({ checks = DEFAULT_CHECKS, runner = runCommand, cwd = PROJECT_ROOT } = {}) {
+  const startedAt = new Date();
+  const results = checks.map((check) => {
+    const run = runner(check.command, { cwd, timeoutMs: check.timeoutMs });
+    return {
+      name: check.name,
+      command: check.command.join(' '),
+      status: run.exitCode === 0 ? 'healthy' : 'unhealthy',
+      exitCode: run.exitCode,
+      durationMs: run.durationMs,
+      error: run.error,
+      outputTail: `${run.stdout}\n${run.stderr}`.trim().slice(-2000),
+    };
+  });
+
+  const healthyCount = results.filter((x) => x.status === 'healthy').length;
+  const unhealthyCount = results.length - healthyCount;
+
+  return {
+    generatedAt: startedAt.toISOString(),
+    durationMs: Date.now() - startedAt.getTime(),
+    overall_status: unhealthyCount === 0 ? 'healthy' : 'unhealthy',
+    summary: {
+      total: results.length,
+      healthy: healthyCount,
+      unhealthy: unhealthyCount,
+    },
+    checks: results,
+  };
+}
+
+function reportToText(report) {
+  const lines = [];
+  lines.push(`Self-Healing Health Check @ ${report.generatedAt}`);
+  lines.push(`Overall: ${report.overall_status.toUpperCase()}`);
+  lines.push(`Checks: ${report.summary.healthy}/${report.summary.total} healthy`);
+  lines.push('');
+
+  report.checks.forEach((check) => {
+    const icon = check.status === 'healthy' ? '✅' : '❌';
+    lines.push(`${icon} ${check.name} (${check.durationMs}ms)`);
+    if (check.status !== 'healthy') {
+      lines.push(`   command: ${check.command}`);
+      if (check.error) lines.push(`   error: ${check.error}`);
+    }
+  });
+
+  return `${lines.join('\n')}\n`;
+}
+
+function runCli() {
+  const args = new Set(process.argv.slice(2));
+  const emitJson = args.has('--json');
+  const noFail = args.has('--no-fail');
+  const report = collectHealthReport();
+
+  if (emitJson) {
+    console.log(JSON.stringify(report, null, 2));
+  } else {
+    process.stdout.write(reportToText(report));
+  }
+
+  if (!noFail && report.overall_status !== 'healthy') {
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  DEFAULT_CHECKS,
+  runCommand,
+  collectHealthReport,
+  reportToText,
+};
+
+if (require.main === module) {
+  runCli();
+}