rlhf-feedback-loop 0.5.0

package/adapters/mcp/server-stdio.js

@@ -0,0 +1,444 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const {
+  captureFeedback,
+  feedbackSummary,
+  analyzeFeedback,
+  writePreventionRules,
+  FEEDBACK_LOG_PATH,
+} = require('../../scripts/feedback-loop');
+const {
+  exportDpoFromMemories,
+  readJSONL,
+  DEFAULT_LOCAL_MEMORY_LOG,
+} = require('../../scripts/export-dpo-pairs');
+const {
+  ensureContextFs,
+  constructContextPack,
+  evaluateContextPack,
+  getProvenance,
+} = require('../../scripts/contextfs');
+const {
+  buildRubricEvaluation,
+} = require('../../scripts/rubric-engine');
+const {
+  listIntents,
+  planIntent,
+} = require('../../scripts/intent-router');
+const {
+  getActiveMcpProfile,
+  getAllowedTools,
+  assertToolAllowed,
+} = require('../../scripts/mcp-policy');
+
+const SERVER_INFO = {
+  name: 'rlhf-feedback-loop-mcp',
+  version: '1.1.0',
+};
+const SAFE_DATA_DIR = path.resolve(path.dirname(FEEDBACK_LOG_PATH));
+
+function resolveSafePath(inputPath, { mustExist = false } = {}) {
+  const allowExternal = process.env.RLHF_ALLOW_EXTERNAL_PATHS === 'true';
+  const resolved = path.resolve(String(inputPath || ''));
+  const inSafeRoot = resolved === SAFE_DATA_DIR || resolved.startsWith(`${SAFE_DATA_DIR}${path.sep}`);
+
+  if (!allowExternal && !inSafeRoot) {
+    throw new Error(`Path must stay within ${SAFE_DATA_DIR}`);
+  }
+  if (mustExist && !fs.existsSync(resolved)) {
+    throw new Error(`Path does not exist: ${resolved}`);
+  }
+  return resolved;
+}
+
+const TOOLS = [
+  {
+    name: 'capture_feedback',
+    description: 'Capture thumbs up/down feedback and promote actionable memory',
+    inputSchema: {
+      type: 'object',
+      required: ['signal', 'context'],
+      properties: {
+        signal: { type: 'string', enum: ['up', 'down'] },
+        context: { type: 'string' },
+        whatWentWrong: { type: 'string' },
+        whatToChange: { type: 'string' },
+        whatWorked: { type: 'string' },
+        tags: { type: 'array', items: { type: 'string' } },
+        skill: { type: 'string' },
+        rubricScores: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              criterion: { type: 'string' },
+              score: { type: 'number' },
+              evidence: { type: 'string' },
+              judge: { type: 'string' },
+            },
+          },
+        },
+        guardrails: {
+          type: 'object',
+          properties: {
+            testsPassed: { type: 'boolean' },
+            pathSafety: { type: 'boolean' },
+            budgetCompliant: { type: 'boolean' },
+          },
+        },
+      },
+    },
+  },
+  {
+    name: 'feedback_summary',
+    description: 'Get summary of recent feedback',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        recent: { type: 'number' },
+      },
+    },
+  },
+  {
+    name: 'feedback_stats',
+    description: 'Get feedback stats and recommendations',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'list_intents',
+    description: 'List available intent plans and whether each requires human approval in the active profile',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        mcpProfile: { type: 'string' },
+        bundleId: { type: 'string' },
+      },
+    },
+  },
+  {
+    name: 'plan_intent',
+    description: 'Generate an intent execution plan with policy checkpoints',
+    inputSchema: {
+      type: 'object',
+      required: ['intentId'],
+      properties: {
+        intentId: { type: 'string' },
+        context: { type: 'string' },
+        mcpProfile: { type: 'string' },
+        bundleId: { type: 'string' },
+        approved: { type: 'boolean' },
+      },
+    },
+  },
+  {
+    name: 'prevention_rules',
+    description: 'Generate prevention rules from repeated mistake patterns',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        minOccurrences: { type: 'number' },
+        outputPath: { type: 'string' },
+      },
+    },
+  },
+  {
+    name: 'export_dpo_pairs',
+    description: 'Export DPO preference pairs from local memory log',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        memoryLogPath: { type: 'string' },
+      },
+    },
+  },
+  {
+    name: 'construct_context_pack',
+    description: 'Construct a bounded context pack from contextfs',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: { type: 'string' },
+        maxItems: { type: 'number' },
+        maxChars: { type: 'number' },
+        namespaces: { type: 'array', items: { type: 'string' } },
+      },
+    },
+  },
+  {
+    name: 'evaluate_context_pack',
+    description: 'Record evaluation outcome for a context pack',
+    inputSchema: {
+      type: 'object',
+      required: ['packId', 'outcome'],
+      properties: {
+        packId: { type: 'string' },
+        outcome: { type: 'string' },
+        signal: { type: 'string' },
+        notes: { type: 'string' },
+        rubricScores: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              criterion: { type: 'string' },
+              score: { type: 'number' },
+              evidence: { type: 'string' },
+              judge: { type: 'string' },
+            },
+          },
+        },
+        guardrails: {
+          type: 'object',
+          properties: {
+            testsPassed: { type: 'boolean' },
+            pathSafety: { type: 'boolean' },
+            budgetCompliant: { type: 'boolean' },
+          },
+        },
+      },
+    },
+  },
+  {
+    name: 'context_provenance',
+    description: 'Get recent context/provenance events',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        limit: { type: 'number' },
+      },
+    },
+  },
+];
+
+function toText(result) {
+  if (typeof result === 'string') return result;
+  return JSON.stringify(result, null, 2);
+}
+
+function parseOptionalObject(input, name) {
+  if (input == null) return {};
+  if (typeof input === 'object' && !Array.isArray(input)) return input;
+  if (typeof input === 'string') {
+    const trimmed = input.trim();
+    if (!trimmed) return {};
+    const parsed = JSON.parse(trimmed);
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      throw new Error(`${name} must be an object`);
+    }
+    return parsed;
+  }
+  throw new Error(`${name} must be an object`);
+}
+
+async function callTool(name, args = {}) {
+  assertToolAllowed(name, getActiveMcpProfile());
+
+  if (name === 'capture_feedback') {
+    const result = captureFeedback({
+      signal: args.signal,
+      context: args.context,
+      whatWentWrong: args.whatWentWrong,
+      whatToChange: args.whatToChange,
+      whatWorked: args.whatWorked,
+      rubricScores: args.rubricScores,
+      guardrails: parseOptionalObject(args.guardrails, 'guardrails'),
+      tags: args.tags || [],
+      skill: args.skill,
+    });
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  if (name === 'feedback_summary') {
+    const recent = Number(args.recent || 20);
+    const summary = feedbackSummary(Number.isFinite(recent) ? recent : 20);
+    return { content: [{ type: 'text', text: summary }] };
+  }
+
+  if (name === 'feedback_stats') {
+    return { content: [{ type: 'text', text: toText(analyzeFeedback()) }] };
+  }
+
+  if (name === 'list_intents') {
+    const result = listIntents({
+      mcpProfile: args.mcpProfile,
+      bundleId: args.bundleId,
+    });
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  if (name === 'plan_intent') {
+    const result = planIntent({
+      intentId: args.intentId,
+      context: args.context || '',
+      mcpProfile: args.mcpProfile,
+      bundleId: args.bundleId,
+      approved: args.approved === true,
+    });
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  if (name === 'prevention_rules') {
+    const minOccurrences = Number(args.minOccurrences || 2);
+    const outputPath = args.outputPath ? resolveSafePath(args.outputPath) : undefined;
+    const result = writePreventionRules(outputPath, Number.isFinite(minOccurrences) ? minOccurrences : 2);
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  if (name === 'export_dpo_pairs') {
+    const memoryLogPath = args.memoryLogPath
+      ? resolveSafePath(args.memoryLogPath, { mustExist: true })
+      : DEFAULT_LOCAL_MEMORY_LOG;
+    const memories = readJSONL(memoryLogPath);
+    const result = exportDpoFromMemories(memories);
+    return {
+      content: [{
+        type: 'text',
+        text: toText({
+          pairs: result.pairs.length,
+          errors: result.errors.length,
+          learnings: result.learnings.length,
+        }),
+      }],
+    };
+  }
+
+  if (name === 'construct_context_pack') {
+    ensureContextFs();
+    const result = constructContextPack({
+      query: args.query || '',
+      maxItems: Number(args.maxItems || 8),
+      maxChars: Number(args.maxChars || 6000),
+      namespaces: Array.isArray(args.namespaces) ? args.namespaces : [],
+    });
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  if (name === 'evaluate_context_pack') {
+    if (!args.packId || !args.outcome) {
+      throw new Error('packId and outcome are required');
+    }
+    const result = evaluateContextPack({
+      packId: args.packId,
+      outcome: args.outcome,
+      signal: args.signal || null,
+      notes: args.notes || '',
+      rubricEvaluation: args.rubricScores || args.guardrails
+        ? buildRubricEvaluation({
+          rubricScores: args.rubricScores,
+          guardrails: parseOptionalObject(args.guardrails, 'guardrails'),
+        })
+        : null,
+    });
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  if (name === 'context_provenance') {
+    const limit = Number(args.limit || 50);
+    const result = getProvenance(Number.isFinite(limit) ? limit : 50);
+    return { content: [{ type: 'text', text: toText(result) }] };
+  }
+
+  throw new Error(`Unknown tool: ${name}`);
+}
+
+async function handleRequest(message) {
+  if (message.method === 'initialize') {
+    return {
+      protocolVersion: '2024-11-05',
+      capabilities: {
+        tools: {},
+      },
+      serverInfo: SERVER_INFO,
+    };
+  }
+
+  if (message.method === 'tools/list') {
+    const profile = getActiveMcpProfile();
+    const allowed = new Set(getAllowedTools(profile));
+    const tools = TOOLS.filter((tool) => allowed.has(tool.name));
+    return { tools };
+  }
+
+  if (message.method === 'tools/call') {
+    const name = message.params && message.params.name;
+    const args = (message.params && message.params.arguments) || {};
+    return callTool(name, args);
+  }
+
+  throw new Error(`Unsupported method: ${message.method}`);
+}
+
+function writeMessage(payload) {
+  const json = JSON.stringify(payload);
+  process.stdout.write(`Content-Length: ${Buffer.byteLength(json, 'utf8')}\r\n\r\n${json}`);
+}
+
+let buffer = Buffer.alloc(0);
+
+function tryReadMessage() {
+  const headerEnd = buffer.indexOf('\r\n\r\n');
+  if (headerEnd === -1) return null;
+
+  const headerRaw = buffer.slice(0, headerEnd).toString('utf8');
+  const match = headerRaw.match(/Content-Length:\s*(\d+)/i);
+  if (!match) {
+    buffer = buffer.slice(headerEnd + 4);
+    return null;
+  }
+
+  const length = Number(match[1]);
+  const totalSize = headerEnd + 4 + length;
+  if (buffer.length < totalSize) return null;
+
+  const body = buffer.slice(headerEnd + 4, totalSize).toString('utf8');
+  buffer = buffer.slice(totalSize);
+
+  return JSON.parse(body);
+}
+
+async function onData(chunk) {
+  buffer = Buffer.concat([buffer, chunk]);
+
+  while (true) {
+    const message = tryReadMessage();
+    if (!message) return;
+
+    if (!Object.prototype.hasOwnProperty.call(message, 'id')) {
+      continue;
+    }
+
+    try {
+      const result = await handleRequest(message);
+      writeMessage({ jsonrpc: '2.0', id: message.id, result });
+    } catch (err) {
+      writeMessage({
+        jsonrpc: '2.0',
+        id: message.id,
+        error: {
+          code: -32603,
+          message: err.message || 'Internal error',
+        },
+      });
+    }
+  }
+}
+
+module.exports = {
+  TOOLS,
+  handleRequest,
+  callTool,
+  resolveSafePath,
+  SAFE_DATA_DIR,
+};
+
+if (require.main === module) {
+  process.stdin.on('data', (chunk) => {
+    onData(chunk).catch((err) => {
+      writeMessage({ jsonrpc: '2.0', id: null, error: { code: -32603, message: err.message } });
+    });
+  });
+}

package/bin/cli.js

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+/**
+ * rlhf-feedback-loop CLI
+ *
+ * Usage:
+ *   npx rlhf-feedback-loop init
+ *
+ * Creates a .rlhf/ directory with config and capture script for local use.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const COMMAND = process.argv[2];
+const CWD = process.cwd();
+
+function init() {
+  const rlhfDir = path.join(CWD, '.rlhf');
+
+  // Create directory
+  if (!fs.existsSync(rlhfDir)) {
+    fs.mkdirSync(rlhfDir, { recursive: true });
+    console.log('Created .rlhf/');
+  } else {
+    console.log('.rlhf/ already exists — updating config');
+  }
+
+  // Write config.json
+  const config = {
+    version: '0.5.0',
+    apiUrl: process.env.RLHF_API_URL || 'http://localhost:3000',
+    logPath: '.rlhf/feedback-log.jsonl',
+    memoryPath: '.rlhf/memory-log.jsonl',
+    createdAt: new Date().toISOString(),
+  };
+
+  const configPath = path.join(rlhfDir, 'config.json');
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
+  console.log('Wrote .rlhf/config.json');
+
+  // Copy capture-feedback script (inline minimal version for standalone use)
+  const captureScript = `#!/usr/bin/env node
+/**
+ * Standalone feedback capture script — created by npx rlhf-feedback-loop init
+ * Full version: https://github.com/IgorGanapolsky/rlhf-feedback-loop
+ *
+ * Usage:
+ *   node .rlhf/capture-feedback.js --feedback=up --context="that worked great" --tags="testing"
+ *   node .rlhf/capture-feedback.js --feedback=down --context="missed edge case" --what-went-wrong="..." --what-to-change="..."
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const CONFIG_PATH = path.join(__dirname, 'config.json');
+const config = fs.existsSync(CONFIG_PATH) ? JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8')) : {};
+const LOG_PATH = path.join(process.cwd(), config.logPath || '.rlhf/feedback-log.jsonl');
+
+function parseArgs(argv) {
+  const args = {};
+  argv.forEach((arg) => {
+    if (!arg.startsWith('--')) return;
+    const [key, ...rest] = arg.slice(2).split('=');
+    args[key] = rest.length ? rest.join('=') : true;
+  });
+  return args;
+}
+
+const args = parseArgs(process.argv.slice(2));
+const signal = args.feedback || args.signal;
+
+if (!signal) {
+  console.error('Error: --feedback=up or --feedback=down required');
+  console.error('Usage: node .rlhf/capture-feedback.js --feedback=up --context="..."');
+  process.exit(1);
+}
+
+const normalized = ['up', 'thumbs_up', 'positive'].includes(signal) ? 'up' : 'down';
+
+const entry = {
+  id: \`fb-\${Date.now()}-\${Math.random().toString(36).slice(2, 7)}\`,
+  signal: normalized,
+  context: args.context || '',
+  whatWentWrong: args['what-went-wrong'] || undefined,
+  whatToChange: args['what-to-change'] || undefined,
+  whatWorked: args['what-worked'] || undefined,
+  tags: args.tags ? args.tags.split(',').map((t) => t.trim()) : [],
+  timestamp: new Date().toISOString(),
+  hostname: os.hostname(),
+};
+
+// Remove undefined fields
+Object.keys(entry).forEach((k) => entry[k] === undefined && delete entry[k]);
+
+// Ensure log directory exists
+const logDir = path.dirname(LOG_PATH);
+if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
+
+fs.appendFileSync(LOG_PATH, JSON.stringify(entry) + '\\n');
+console.log(\`Feedback captured [\${normalized}]: \${entry.id}\`);
+console.log(\`Logged to: \${LOG_PATH}\`);
+`;
+
+  const scriptPath = path.join(rlhfDir, 'capture-feedback.js');
+  fs.writeFileSync(scriptPath, captureScript);
+  // Make executable
+  try {
+    fs.chmodSync(scriptPath, '755');
+  } catch (_) {
+    // chmod may not be available on all platforms — not fatal
+  }
+  console.log('Wrote .rlhf/capture-feedback.js');
+
+  // Add .rlhf/feedback-log.jsonl to .gitignore if it exists
+  const gitignorePath = path.join(CWD, '.gitignore');
+  if (fs.existsSync(gitignorePath)) {
+    const gitignore = fs.readFileSync(gitignorePath, 'utf8');
+    const entries = ['.rlhf/feedback-log.jsonl', '.rlhf/memory-log.jsonl'];
+    const missing = entries.filter((e) => !gitignore.includes(e));
+    if (missing.length > 0) {
+      fs.appendFileSync(gitignorePath, '\n# RLHF local feedback data\n' + missing.join('\n') + '\n');
+      console.log('Updated .gitignore with RLHF data paths');
+    }
+  }
+
+  console.log('');
+  console.log('Setup complete! Run:');
+  console.log("  node .rlhf/capture-feedback.js --feedback=up --context='test'");
+  console.log('');
+  console.log('Full docs: https://github.com/IgorGanapolsky/rlhf-feedback-loop');
+}
+
+function help() {
+  console.log('rlhf-feedback-loop CLI');
+  console.log('');
+  console.log('Commands:');
+  console.log('  init    Scaffold .rlhf/ config and capture script in current directory');
+  console.log('  help    Show this help message');
+  console.log('');
+  console.log('Examples:');
+  console.log('  npx rlhf-feedback-loop init');
+  console.log('  node .rlhf/capture-feedback.js --feedback=up --context="great result"');
+}
+
+switch (COMMAND) {
+  case 'init':
+    init();
+    break;
+  case 'help':
+  case '--help':
+  case '-h':
+    help();
+    break;
+  default:
+    if (COMMAND) {
+      console.error(`Unknown command: ${COMMAND}`);
+      console.error('Run: npx rlhf-feedback-loop help');
+      process.exit(1);
+    } else {
+      help();
+    }
+}

package/config/mcp-allowlists.json

@@ -0,0 +1,29 @@
+{
+  "version": 1,
+  "profiles": {
+    "default": [
+      "capture_feedback",
+      "feedback_summary",
+      "feedback_stats",
+      "list_intents",
+      "plan_intent",
+      "prevention_rules",
+      "export_dpo_pairs",
+      "construct_context_pack",
+      "evaluate_context_pack",
+      "context_provenance"
+    ],
+    "readonly": [
+      "feedback_summary",
+      "feedback_stats",
+      "list_intents",
+      "plan_intent",
+      "context_provenance"
+    ],
+    "locked": [
+      "feedback_summary",
+      "list_intents",
+      "plan_intent"
+    ]
+  }
+}

package/config/policy-bundles/constrained-v1.json

@@ -0,0 +1,53 @@
+{
+  "bundleId": "constrained-v1",
+  "version": 1,
+  "description": "Conservative execution bundle for locked and review-heavy environments.",
+  "defaultMcpProfile": "locked",
+  "approval": {
+    "requiredRisks": ["medium", "high", "critical"],
+    "profileOverrides": {
+      "default": ["medium", "high", "critical"],
+      "readonly": ["medium", "high", "critical"],
+      "locked": ["low", "medium", "high", "critical"]
+    }
+  },
+  "intents": [
+    {
+      "id": "capture_feedback_loop",
+      "description": "Capture user outcome and update memory artifacts.",
+      "risk": "low",
+      "actions": [
+        {
+          "kind": "mcp_tool",
+          "name": "capture_feedback"
+        }
+      ]
+    },
+    {
+      "id": "improve_response_quality",
+      "description": "Summarize recent failures and regenerate prevention rules.",
+      "risk": "medium",
+      "actions": [
+        {
+          "kind": "mcp_tool",
+          "name": "feedback_summary"
+        },
+        {
+          "kind": "mcp_tool",
+          "name": "prevention_rules"
+        }
+      ]
+    },
+    {
+      "id": "publish_dpo_training_data",
+      "description": "Export DPO preference pairs for model improvement pipelines.",
+      "risk": "high",
+      "actions": [
+        {
+          "kind": "mcp_tool",
+          "name": "export_dpo_pairs"
+        }
+      ]
+    }
+  ]
+}