rlhf-feedback-loop 0.5.0

package/scripts/intent-router.js

@@ -0,0 +1,164 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { getActiveMcpProfile, getAllowedTools } = require('./mcp-policy');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_BUNDLE_DIR = path.join(PROJECT_ROOT, 'config', 'policy-bundles');
+const RISK_LEVELS = ['low', 'medium', 'high', 'critical'];
+
+function getDefaultBundleId() {
+  return process.env.RLHF_POLICY_BUNDLE || 'default-v1';
+}
+
+function getBundlePath(bundleId = getDefaultBundleId()) {
+  if (process.env.RLHF_POLICY_BUNDLE_PATH) {
+    return process.env.RLHF_POLICY_BUNDLE_PATH;
+  }
+  return path.join(DEFAULT_BUNDLE_DIR, `${bundleId}.json`);
+}
+
+function validateBundle(bundle) {
+  if (!bundle || typeof bundle !== 'object') {
+    throw new Error('Invalid policy bundle: expected object');
+  }
+  if (!bundle.bundleId || typeof bundle.bundleId !== 'string') {
+    throw new Error('Invalid policy bundle: missing bundleId');
+  }
+  if (!Array.isArray(bundle.intents) || bundle.intents.length === 0) {
+    throw new Error('Invalid policy bundle: intents must be a non-empty array');
+  }
+
+  bundle.intents.forEach((intent) => {
+    if (!intent.id || typeof intent.id !== 'string') {
+      throw new Error('Invalid policy bundle: intent id is required');
+    }
+    if (!RISK_LEVELS.includes(intent.risk)) {
+      throw new Error(`Invalid policy bundle: unsupported risk '${intent.risk}' for intent '${intent.id}'`);
+    }
+    if (!Array.isArray(intent.actions) || intent.actions.length === 0) {
+      throw new Error(`Invalid policy bundle: intent '${intent.id}' must define actions`);
+    }
+  });
+
+  return true;
+}
+
+function loadPolicyBundle(bundleId = getDefaultBundleId()) {
+  const raw = fs.readFileSync(getBundlePath(bundleId), 'utf-8');
+  const parsed = JSON.parse(raw);
+  validateBundle(parsed);
+  return parsed;
+}
+
+function getRequiredApprovalRisks(bundle, mcpProfile) {
+  const approval = bundle.approval || {};
+  if (approval.profileOverrides && Array.isArray(approval.profileOverrides[mcpProfile])) {
+    return approval.profileOverrides[mcpProfile];
+  }
+  return Array.isArray(approval.requiredRisks) ? approval.requiredRisks : ['high', 'critical'];
+}
+
+function assertKnownMcpProfile(profile) {
+  getAllowedTools(profile);
+  return profile;
+}
+
+function listIntents(options = {}) {
+  const bundle = loadPolicyBundle(options.bundleId);
+  const profile = assertKnownMcpProfile(options.mcpProfile || getActiveMcpProfile());
+  const requiredRisks = getRequiredApprovalRisks(bundle, profile);
+
+  return {
+    bundleId: bundle.bundleId,
+    mcpProfile: profile,
+    intents: bundle.intents.map((intent) => ({
+      id: intent.id,
+      description: intent.description,
+      risk: intent.risk,
+      actionCount: intent.actions.length,
+      requiresApproval: requiredRisks.includes(intent.risk),
+    })),
+  };
+}
+
+function planIntent(options = {}) {
+  const bundle = loadPolicyBundle(options.bundleId);
+  const profile = assertKnownMcpProfile(options.mcpProfile || getActiveMcpProfile());
+  const intentId = String(options.intentId || '').trim();
+  const context = String(options.context || '').trim();
+  const approved = options.approved === true;
+
+  if (!intentId) {
+    throw new Error('intentId is required');
+  }
+
+  const intent = bundle.intents.find((item) => item.id === intentId);
+  if (!intent) {
+    throw new Error(`Unknown intent: ${intentId}`);
+  }
+
+  const requiredRisks = getRequiredApprovalRisks(bundle, profile);
+  const requiresApproval = requiredRisks.includes(intent.risk);
+  const checkpointRequired = requiresApproval && !approved;
+
+  return {
+    bundleId: bundle.bundleId,
+    mcpProfile: profile,
+    generatedAt: new Date().toISOString(),
+    status: checkpointRequired ? 'checkpoint_required' : 'ready',
+    intent: {
+      id: intent.id,
+      description: intent.description,
+      risk: intent.risk,
+    },
+    context,
+    requiresApproval,
+    approved,
+    checkpoint: checkpointRequired
+      ? {
+        type: 'human_approval',
+        reason: `Intent '${intent.id}' has risk '${intent.risk}' under profile '${profile}'.`,
+        requiredForRiskLevels: requiredRisks,
+      }
+      : null,
+    actions: intent.actions,
+  };
+}
+
+module.exports = {
+  DEFAULT_BUNDLE_DIR,
+  RISK_LEVELS,
+  getDefaultBundleId,
+  getBundlePath,
+  validateBundle,
+  loadPolicyBundle,
+  getRequiredApprovalRisks,
+  assertKnownMcpProfile,
+  listIntents,
+  planIntent,
+};
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const intentArg = args.find((arg) => arg.startsWith('--intent='));
+  const profileArg = args.find((arg) => arg.startsWith('--profile='));
+  const bundleArg = args.find((arg) => arg.startsWith('--bundle='));
+  const approved = args.includes('--approved');
+
+  if (!intentArg) {
+    console.log(JSON.stringify(listIntents({
+      mcpProfile: profileArg ? profileArg.replace('--profile=', '') : undefined,
+      bundleId: bundleArg ? bundleArg.replace('--bundle=', '') : undefined,
+    }), null, 2));
+    process.exit(0);
+  }
+
+  const plan = planIntent({
+    intentId: intentArg.replace('--intent=', ''),
+    mcpProfile: profileArg ? profileArg.replace('--profile=', '') : undefined,
+    bundleId: bundleArg ? bundleArg.replace('--bundle=', '') : undefined,
+    approved,
+  });
+  console.log(JSON.stringify(plan, null, 2));
+}

package/scripts/mcp-policy.js

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_POLICY_PATH = path.join(PROJECT_ROOT, 'config', 'mcp-allowlists.json');
+const DEFAULT_SUBAGENT_PROFILE_PATH = path.join(PROJECT_ROOT, 'config', 'subagent-profiles.json');
+
+function getPolicyPath() {
+  return process.env.RLHF_MCP_POLICY_PATH || DEFAULT_POLICY_PATH;
+}
+
+function loadMcpPolicy() {
+  const policyPath = getPolicyPath();
+  const raw = fs.readFileSync(policyPath, 'utf-8');
+  const parsed = JSON.parse(raw);
+  if (!parsed.profiles || typeof parsed.profiles !== 'object') {
+    throw new Error('Invalid MCP policy: missing profiles object');
+  }
+  return parsed;
+}
+
+function loadSubagentProfiles() {
+  const profilePath = process.env.RLHF_SUBAGENT_PROFILE_PATH || DEFAULT_SUBAGENT_PROFILE_PATH;
+  const raw = fs.readFileSync(profilePath, 'utf-8');
+  const parsed = JSON.parse(raw);
+  if (!parsed.profiles || typeof parsed.profiles !== 'object') {
+    throw new Error('Invalid subagent profile config: missing profiles object');
+  }
+  return parsed;
+}
+
+function getActiveMcpProfile() {
+  const explicitProfile = process.env.RLHF_MCP_PROFILE || null;
+  const runtimeSubagentProfile = process.env.RLHF_SUBAGENT_PROFILE || null;
+
+  if (!runtimeSubagentProfile) {
+    return explicitProfile || 'default';
+  }
+
+  const config = loadSubagentProfiles();
+  const subagent = config.profiles[runtimeSubagentProfile];
+  if (!subagent || !subagent.mcpProfile) {
+    throw new Error(`Unknown subagent profile: ${runtimeSubagentProfile}`);
+  }
+
+  if (explicitProfile && explicitProfile !== subagent.mcpProfile) {
+    throw new Error(
+      `MCP profile conflict: RLHF_MCP_PROFILE='${explicitProfile}' does not match subagent profile '${runtimeSubagentProfile}' (${subagent.mcpProfile})`,
+    );
+  }
+
+  return subagent.mcpProfile;
+}
+
+function getAllowedTools(profileName = getActiveMcpProfile()) {
+  const policy = loadMcpPolicy();
+  const tools = policy.profiles[profileName];
+  if (!tools) {
+    throw new Error(`Unknown MCP profile: ${profileName}`);
+  }
+  return tools;
+}
+
+function isToolAllowed(toolName, profileName = getActiveMcpProfile()) {
+  const allowed = getAllowedTools(profileName);
+  return allowed.includes(toolName);
+}
+
+function assertToolAllowed(toolName, profileName = getActiveMcpProfile()) {
+  if (!isToolAllowed(toolName, profileName)) {
+    throw new Error(`Tool '${toolName}' is not allowed in MCP profile '${profileName}'`);
+  }
+}
+
+module.exports = {
+  DEFAULT_POLICY_PATH,
+  getPolicyPath,
+  loadMcpPolicy,
+  loadSubagentProfiles,
+  getActiveMcpProfile,
+  getAllowedTools,
+  isToolAllowed,
+  assertToolAllowed,
+  DEFAULT_SUBAGENT_PROFILE_PATH,
+};
+
+if (require.main === module) {
+  const profile = getActiveMcpProfile();
+  const tools = getAllowedTools(profile);
+  console.log(JSON.stringify({ profile, tools }, null, 2));
+}

package/scripts/meta-policy.js

@@ -0,0 +1,194 @@
+'use strict';
+
+/**
+ * Meta-Policy Rule Extraction (DPO-03)
+ *
+ * Reads memory-log.jsonl, groups negative memories by domain, computes
+ * recency-weighted confidence scores, detects trend direction, and writes
+ * meta-policy-rules.json.
+ *
+ * Output file: {RLHF_FEEDBACK_DIR}/meta-policy-rules.json
+ *
+ * This is a different artifact from prevention-rules.md (simpler occurrence counts).
+ * Meta-policy rules have confidence + trend + recency weighting.
+ *
+ * Min occurrences threshold: 2 (consistent with buildPreventionRules())
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { parseTimestamp } = require('./feedback-schema');
+const { timeDecayWeight } = require('./thompson-sampling');
+const { inferDomain } = require('./feedback-loop');
+
+const MIN_OCCURRENCES = 2;
+const RECENT_DAYS = 7;
+const RECENT_MS = RECENT_DAYS * 24 * 3600 * 1000;
+
+/**
+ * Extract meta-policy rules from memory-log.jsonl feedback trends.
+ *
+ * @param {object} opts
+ * @param {string} [opts.feedbackDir] - Override feedback directory (default: RLHF_FEEDBACK_DIR or ~/.claude/memory/feedback)
+ * @returns {Array<{category: string, confidence: number, trend: string, occurrence_count: number, last_seen: string}>}
+ */
+function extractMetaPolicyRules(opts = {}) {
+  const feedbackDir = opts.feedbackDir
+    || process.env.RLHF_FEEDBACK_DIR
+    || path.join(os.homedir(), '.claude', 'memory', 'feedback');
+
+  const memoryLogPath = path.join(feedbackDir, 'memory-log.jsonl');
+
+  if (!fs.existsSync(memoryLogPath)) {
+    return [];
+  }
+
+  const raw = fs.readFileSync(memoryLogPath, 'utf-8').trim();
+  if (!raw) {
+    return [];
+  }
+
+  // Parse all entries, skip malformed lines
+  const allEntries = raw.split('\n').reduce((acc, line) => {
+    if (!line.trim()) return acc;
+    try {
+      acc.push(JSON.parse(line));
+    } catch {
+      process.stderr.write(`meta-policy: skipping malformed line: ${line.slice(0, 80)}\n`);
+    }
+    return acc;
+  }, []);
+
+  if (allEntries.length === 0) {
+    return [];
+  }
+
+  // Filter to negative memories only
+  const negativeEntries = allEntries.filter(
+    (e) => e.signal === 'negative' || e.feedback === 'down',
+  );
+
+  if (negativeEntries.length === 0) {
+    return [];
+  }
+
+  // Group negative entries by domain
+  const domainMap = new Map();
+  for (const entry of negativeEntries) {
+    const domain = inferDomain(entry.tags, entry.context);
+    if (!domainMap.has(domain)) {
+      domainMap.set(domain, []);
+    }
+    domainMap.get(domain).push(entry);
+  }
+
+  // Build rules for domains with enough occurrences
+  const now = Date.now();
+  const rules = [];
+
+  for (const [domain, entries] of domainMap) {
+    if (entries.length < MIN_OCCURRENCES) {
+      continue;
+    }
+
+    // Compute avg time-decay weight across all negative entries
+    const weights = entries.map((e) => timeDecayWeight(e.timestamp));
+    const avg_weighted = weights.reduce((sum, w) => sum + w, 0) / weights.length;
+
+    // Count recent negative entries (last 7 days)
+    const recent_entries = entries.filter((e) => {
+      const ts = parseTimestamp(e.timestamp);
+      return ts && (now - ts.getTime()) < RECENT_MS;
+    }).length;
+
+    // Count recent positive entries for same domain (from full allEntries log)
+    const recent_positive = allEntries.filter((e) => {
+      if (e.signal !== 'positive' && e.feedback !== 'up') return false;
+      const entryDomain = inferDomain(e.tags, e.context);
+      if (entryDomain !== domain) return false;
+      const ts = parseTimestamp(e.timestamp);
+      return ts && (now - ts.getTime()) < RECENT_MS;
+    }).length;
+
+    // Compute confidence: min(0.95, 0.4 + (avg_weighted * 0.3) + (occurrence_count * 0.05))
+    const confidence = Math.min(
+      0.95,
+      0.4 + (avg_weighted * 0.3) + (entries.length * 0.05),
+    );
+
+    // Determine trend
+    let trend;
+    if (recent_entries === 0 && recent_positive > 0) {
+      trend = 'improving';
+    } else if (recent_entries > 2 && recent_positive === 0) {
+      trend = 'deteriorating';
+    } else if (recent_entries > recent_positive) {
+      trend = 'needs_attention';
+    } else {
+      trend = 'stable';
+    }
+
+    // Find most recent entry timestamp
+    const timestamps = entries
+      .map((e) => parseTimestamp(e.timestamp))
+      .filter(Boolean)
+      .sort((a, b) => b.getTime() - a.getTime());
+    const last_seen = timestamps.length > 0
+      ? timestamps[0].toISOString()
+      : new Date().toISOString();
+
+    rules.push({
+      category: domain,
+      confidence: Math.round(confidence * 1000) / 1000,
+      trend,
+      occurrence_count: entries.length,
+      last_seen,
+    });
+  }
+
+  // Sort by confidence descending (most urgent first)
+  rules.sort((a, b) => b.confidence - a.confidence);
+
+  return rules;
+}
+
+/**
+ * Run meta-policy rule extraction and write results to meta-policy-rules.json.
+ *
+ * @param {object} opts - Same as extractMetaPolicyRules opts
+ * @returns {{ rules: Array, outputPath: string }}
+ */
+function run(opts = {}) {
+  const feedbackDir = opts.feedbackDir
+    || process.env.RLHF_FEEDBACK_DIR
+    || path.join(os.homedir(), '.claude', 'memory', 'feedback');
+
+  const rules = extractMetaPolicyRules({ ...opts, feedbackDir });
+
+  // Ensure output directory exists
+  if (!fs.existsSync(feedbackDir)) {
+    fs.mkdirSync(feedbackDir, { recursive: true });
+  }
+
+  // Write to meta-policy-rules.json — NOT to prevention-rules.md (see RESEARCH.md Pitfall 4)
+  const outputPath = path.join(feedbackDir, 'meta-policy-rules.json');
+  fs.writeFileSync(
+    outputPath,
+    JSON.stringify({ generated: new Date().toISOString(), rules }, null, 2),
+  );
+
+  console.log(`meta-policy: extracted ${rules.length} rules`);
+  return { rules, outputPath };
+}
+
+module.exports = { extractMetaPolicyRules, run };
+
+if (require.main === module && process.argv.includes('--extract')) {
+  try {
+    run();
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+}

package/scripts/plan-gate.js

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// ---------------------------------------------------------------------------
+// Gate validators
+// ---------------------------------------------------------------------------
+
+function countTableRows(content, sectionHeading) {
+  const sectionRegex = new RegExp(
+    `#+\\s*${sectionHeading}[^\\n]*\\n([\\s\\S]*?)(?=\\n#+\\s|$)`,
+  );
+  const match = content.match(sectionRegex);
+  if (!match) return 0;
+
+  const lines = match[1].split('\n').filter((l) => l.trim().startsWith('|'));
+  // Subtract header row and separator row
+  const dataRows = lines.filter(
+    (l) => !/^\|\s*-+/.test(l.trim()) && !/^\|\s*:?-+/.test(l.trim()),
+  );
+  // First row is the header
+  return Math.max(0, dataRows.length - 1);
+}
+
+function countContracts(content) {
+  const sectionRegex = /#+\s*Contracts[^\n]*\n([\s\S]*?)(?=\n#+\s|$)/;
+  const match = content.match(sectionRegex);
+  if (!match) return 0;
+
+  const section = match[1];
+  // Find code blocks and look for interface/type keywords inside them
+  const codeBlockRegex = /```[\s\S]*?```/g;
+  let count = 0;
+  let blockMatch;
+  while ((blockMatch = codeBlockRegex.exec(section)) !== null) {
+    const block = blockMatch[0];
+    const interfaceMatches = block.match(/\b(interface|type)\s+\w+/g);
+    if (interfaceMatches) count += interfaceMatches.length;
+  }
+  return count;
+}
+
+function countValidationScenarios(content) {
+  const sectionRegex =
+    /#+\s*Validation\s+Checklist[^\n]*\n([\s\S]*?)(?=\n#+\s|$)/;
+  const match = content.match(sectionRegex);
+  if (!match) return 0;
+
+  const lines = match[1].split('\n');
+  return lines.filter((l) => /^\s*-\s*\[\s*\]/.test(l)).length;
+}
+
+function getStatus(content) {
+  const match = content.match(/#+\s*Status[^\n]*\n\s*(\S+)/);
+  return match ? match[1].trim() : null;
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+function validatePlan(content) {
+  const questionCount = countTableRows(content, 'Clarifying Questions Resolved');
+  const contractCount = countContracts(content);
+  const scenarioCount = countValidationScenarios(content);
+  const status = getStatus(content);
+
+  const gates = [
+    {
+      name: 'Clarifying Questions',
+      pass: questionCount >= 3,
+      detail: `${questionCount} questions resolved`,
+    },
+    {
+      name: 'Contracts Defined',
+      pass: contractCount >= 1,
+      detail: `${contractCount} interface${contractCount !== 1 ? 's' : ''} found`,
+    },
+    {
+      name: 'Validation Checklist',
+      pass: scenarioCount >= 2,
+      detail: `${scenarioCount} scenarios defined`,
+    },
+    {
+      name: 'Status',
+      pass: status !== 'COMPLETE',
+      detail:
+        status === 'COMPLETE'
+          ? 'COMPLETE (already finished — cannot re-approve)'
+          : `${status || 'UNKNOWN'} (not COMPLETE)`,
+    },
+  ];
+
+  const allPass = gates.every((g) => g.pass);
+  return { gates, allPass };
+}
+
+function formatReport(result) {
+  const lines = result.gates.map(
+    (g) => `${g.pass ? '✅' : '❌'} ${g.name}: ${g.detail}`,
+  );
+  lines.push('');
+  lines.push(
+    result.allPass
+      ? 'RESULT: PASS — all gates satisfied'
+      : 'RESULT: BLOCKED — resolve issues above before spawning agents',
+  );
+  return lines.join('\n');
+}
+
+function run() {
+  const args = process.argv.slice(2);
+  const jsonFlag = args.includes('--json');
+  const filePath = args.find((a) => a !== '--json');
+
+  if (!filePath) {
+    console.error('Usage: node scripts/plan-gate.js <plan-file.md> [--json]');
+    process.exit(1);
+  }
+
+  const resolved = path.resolve(filePath);
+  if (!fs.existsSync(resolved)) {
+    console.error(`File not found: ${resolved}`);
+    process.exit(1);
+  }
+
+  const content = fs.readFileSync(resolved, 'utf-8');
+  const result = validatePlan(content);
+
+  if (jsonFlag) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    console.log(formatReport(result));
+  }
+
+  process.exit(result.allPass ? 0 : 1);
+}
+
+// Export for testing
+module.exports = {
+  validatePlan,
+  formatReport,
+  countTableRows,
+  countContracts,
+  countValidationScenarios,
+  getStatus,
+};
+
+// Run only when executed directly
+if (require.main === module) {
+  run();
+}