rlhf-feedback-loop 0.5.0

package/scripts/feedback-schema.js

@@ -0,0 +1,300 @@
+#!/usr/bin/env node
+/**
+ * Feedback Schema Validator
+ *
+ * Implements three reliability patterns:
+ *   1. Typed schemas — enforce structure on every feedback memory
+ *   2. Action schemas — discriminated union of allowed feedback actions
+ *   3. Validation at boundaries — reject bad data before storage
+ */
+
+const GENERIC_TAGS = new Set(['feedback', 'positive', 'negative']);
+const MIN_CONTENT_LENGTH = 20;
+const VALID_TITLE_PREFIXES = ['SUCCESS:', 'MISTAKE:', 'LEARNING:', 'PREFERENCE:'];
+const VALID_CATEGORIES = new Set(['error', 'learning', 'preference']);
+
+function validateFeedbackMemory(memory) {
+  const issues = [];
+
+  if (!memory.title || typeof memory.title !== 'string') {
+    issues.push('title: required string');
+  } else {
+    const hasPrefix = VALID_TITLE_PREFIXES.some((p) => memory.title.startsWith(p));
+    if (!hasPrefix) {
+      issues.push(`title: must start with one of ${VALID_TITLE_PREFIXES.join(', ')}`);
+    }
+    const afterPrefix = memory.title.replace(/^(SUCCESS|MISTAKE|LEARNING|PREFERENCE):\s*/, '');
+    if (afterPrefix.length < 5) {
+      issues.push('title: description after prefix too short (min 5 chars)');
+    }
+  }
+
+  if (!memory.content || typeof memory.content !== 'string') {
+    issues.push('content: required string');
+  } else if (memory.content.length < MIN_CONTENT_LENGTH) {
+    issues.push(`content: too short (${memory.content.length} chars, min ${MIN_CONTENT_LENGTH})`);
+  }
+
+  if (!memory.category) {
+    issues.push('category: required');
+  } else if (!VALID_CATEGORIES.has(memory.category)) {
+    issues.push(`category: must be one of ${[...VALID_CATEGORIES].join(', ')} (got "${memory.category}")`);
+  }
+
+  if (!Array.isArray(memory.tags) || memory.tags.length === 0) {
+    issues.push('tags: at least 1 tag required');
+  } else {
+    const domainTags = memory.tags.filter((t) => !GENERIC_TAGS.has(t));
+    if (domainTags.length === 0) {
+      issues.push('tags: at least 1 non-generic tag required');
+    }
+  }
+
+  if (memory.title && memory.category) {
+    const titleIsError = memory.title.startsWith('MISTAKE:');
+    const titleIsSuccess = memory.title.startsWith('SUCCESS:') || memory.title.startsWith('LEARNING:');
+    if (titleIsError && memory.category !== 'error') {
+      issues.push('consistency: MISTAKE title should have category "error"');
+    }
+    if (titleIsSuccess && memory.category === 'error') {
+      issues.push('consistency: SUCCESS/LEARNING title should not have category "error"');
+    }
+  }
+
+  if (memory.rubricSummary != null) {
+    if (typeof memory.rubricSummary !== 'object') {
+      issues.push('rubricSummary: must be an object when provided');
+    } else {
+      const weightedScore = Number(memory.rubricSummary.weightedScore);
+      if (!Number.isFinite(weightedScore) || weightedScore < 0 || weightedScore > 1) {
+        issues.push('rubricSummary.weightedScore: must be a number between 0 and 1');
+      }
+      if (!Array.isArray(memory.rubricSummary.failingCriteria)) {
+        issues.push('rubricSummary.failingCriteria: must be an array');
+      }
+      if (!Array.isArray(memory.rubricSummary.failingGuardrails)) {
+        issues.push('rubricSummary.failingGuardrails: must be an array');
+      }
+    }
+  }
+
+  return { valid: issues.length === 0, issues };
+}
+
+function resolveFeedbackAction(params) {
+  const {
+    signal,
+    context,
+    whatWentWrong,
+    whatToChange,
+    whatWorked,
+    tags,
+    rubricEvaluation,
+  } = params;
+
+  if (!context && !whatWentWrong && !whatWorked) {
+    return { type: 'no-action', reason: 'No context provided — cannot create actionable memory' };
+  }
+
+  const domainTags = (tags || []).filter((t) => !GENERIC_TAGS.has(t));
+  const rubricSummary = rubricEvaluation
+    ? {
+      rubricId: rubricEvaluation.rubricId,
+      weightedScore: rubricEvaluation.weightedScore,
+      failingCriteria: rubricEvaluation.failingCriteria || [],
+      failingGuardrails: rubricEvaluation.failingGuardrails || [],
+      judgeDisagreements: rubricEvaluation.judgeDisagreements || [],
+      blockReasons: rubricEvaluation.blockReasons || [],
+    }
+    : null;
+  const rubricFailureTags = rubricSummary
+    ? (rubricSummary.failingCriteria || []).map((criterion) => `rubric-${criterion}`)
+    : [];
+
+  if (signal === 'negative') {
+    if (!whatWentWrong && !context) {
+      return { type: 'no-action', reason: 'Negative feedback without context — cannot determine what went wrong' };
+    }
+
+    const content = [
+      whatWentWrong ? `What went wrong: ${whatWentWrong}` : `Context: ${context}`,
+      whatToChange ? `How to avoid: ${whatToChange}` : 'Action needed: investigate and prevent recurrence',
+    ].join('\n');
+    const rubricLines = [];
+    if (rubricSummary) {
+      rubricLines.push(`Rubric weighted score: ${rubricSummary.weightedScore}`);
+      if (rubricSummary.failingCriteria.length > 0) {
+        rubricLines.push(`Rubric failing criteria: ${rubricSummary.failingCriteria.join(', ')}`);
+      }
+      if (rubricSummary.failingGuardrails.length > 0) {
+        rubricLines.push(`Guardrails failed: ${rubricSummary.failingGuardrails.join(', ')}`);
+      }
+      if (rubricSummary.judgeDisagreements.length > 0) {
+        rubricLines.push('Judge disagreement detected; require manual review');
+      }
+    }
+
+    const description = whatWentWrong ? whatWentWrong.slice(0, 60) : (context || '').slice(0, 60);
+
+    return {
+      type: 'store-mistake',
+      memory: {
+        title: `MISTAKE: ${description}`,
+        content: rubricLines.length > 0 ? `${content}\n${rubricLines.join('\n')}` : content,
+        category: 'error',
+        importance: 'high',
+        tags: ['feedback', 'negative', ...domainTags, ...rubricFailureTags],
+        rubricSummary,
+      },
+    };
+  }
+
+  if (signal === 'positive') {
+    if (rubricEvaluation && !rubricEvaluation.promotionEligible) {
+      const reasons = rubricEvaluation.blockReasons && rubricEvaluation.blockReasons.length > 0
+        ? rubricEvaluation.blockReasons.join('; ')
+        : 'rubric gate did not pass';
+      return { type: 'no-action', reason: `Rubric gate prevented promotion: ${reasons}` };
+    }
+
+    if (!whatWorked && !context) {
+      return { type: 'no-action', reason: 'Positive feedback without context — cannot determine what worked' };
+    }
+
+    const content = whatWorked ? `What worked: ${whatWorked}` : `Approach: ${context}`;
+    const rubricLines = [];
+    if (rubricSummary) {
+      rubricLines.push(`Rubric weighted score: ${rubricSummary.weightedScore}`);
+      rubricLines.push(`Rubric criteria passed with no blocking guardrails.`);
+    }
+    const description = whatWorked ? whatWorked.slice(0, 60) : (context || '').slice(0, 60);
+
+    return {
+      type: 'store-learning',
+      memory: {
+        title: `SUCCESS: ${description}`,
+        content: rubricLines.length > 0 ? `${content}\n${rubricLines.join('\n')}` : content,
+        category: 'learning',
+        importance: 'normal',
+        tags: ['feedback', 'positive', ...domainTags],
+        rubricSummary,
+      },
+    };
+  }
+
+  return { type: 'no-action', reason: `Unknown signal: ${signal}` };
+}
+
+function prepareForStorage(memory) {
+  const validation = validateFeedbackMemory(memory);
+  if (!validation.valid) {
+    return { ok: false, issues: validation.issues };
+  }
+  return { ok: true, memory };
+}
+
+/**
+ * parseTimestamp — Parse any ISO 8601 timestamp string into a Date object.
+ * Handles: Z-suffix ("2026-03-04T12:00:00.000Z"), no-suffix ("2026-03-04T12:00:00"),
+ * and UTC offset ("2026-03-04T12:00:00+05:00").
+ * Returns null (not NaN) for null, undefined, or unparseable input.
+ * NOTE: Do NOT change how timestamps are WRITTEN — new Date().toISOString() already
+ * produces correct ISO 8601+Z format. This helper is for READING only.
+ * Python's train_from_feedback.py strips Z with .replace("Z","") before fromisoformat().
+ * That pattern is safe because Node always writes Z-suffix. Do not alter write behavior.
+ * @param {string|null|undefined} ts - Timestamp string to parse
+ * @returns {Date|null}
+ */
+function parseTimestamp(ts) {
+  if (ts == null) return null;
+  const d = new Date(String(ts).trim());
+  return isNaN(d.getTime()) ? null : d;
+}
+
+function runTests() {
+  let passed = 0;
+  let failed = 0;
+
+  function assert(condition, name) {
+    if (condition) {
+      passed++;
+      console.log(`  PASS ${name}`);
+    } else {
+      failed++;
+      console.log(`  FAIL ${name}`);
+    }
+  }
+
+  console.log('\nfeedback-schema.js tests\n');
+
+  const goodError = {
+    title: 'MISTAKE: Did not verify before claiming fixed',
+    content: 'Always run tests and show evidence before claiming the work is complete.',
+    category: 'error',
+    tags: ['feedback', 'negative', 'verification'],
+  };
+  assert(validateFeedbackMemory(goodError).valid, 'valid error memory passes');
+
+  const shortContent = {
+    title: 'MISTAKE: Bad fix regression',
+    content: 'thumbs down',
+    category: 'error',
+    tags: ['verification'],
+  };
+  assert(!validateFeedbackMemory(shortContent).valid, 'short content fails');
+
+  const bareThumbsDown = resolveFeedbackAction({ signal: 'negative' });
+  assert(bareThumbsDown.type === 'no-action', 'bare negative feedback becomes no-action');
+
+  const fullNegative = resolveFeedbackAction({
+    signal: 'negative',
+    context: 'Pushed code with no tests',
+    whatWentWrong: 'Claimed fixed without test output',
+    whatToChange: 'Always run tests first',
+    tags: ['testing', 'verification'],
+  });
+  assert(fullNegative.type === 'store-mistake', 'negative feedback creates store-mistake action');
+
+  const prep = prepareForStorage(fullNegative.memory);
+  assert(prep.ok, 'store-mistake memory passes storage validation');
+
+  const fullPositive = resolveFeedbackAction({
+    signal: 'positive',
+    whatWorked: 'Ran tests and included output before final response',
+    tags: ['testing', 'verification'],
+  });
+  assert(fullPositive.type === 'store-learning', 'positive feedback creates store-learning action');
+
+  const blockedPositive = resolveFeedbackAction({
+    signal: 'positive',
+    whatWorked: 'Looked correct',
+    tags: ['testing'],
+    rubricEvaluation: {
+      promotionEligible: false,
+      blockReasons: ['failed_guardrails:testsPassed'],
+      failingCriteria: [],
+      failingGuardrails: ['testsPassed'],
+      weightedScore: 0.82,
+      rubricId: 'default-v1',
+    },
+  });
+  assert(blockedPositive.type === 'no-action', 'rubric gate blocks unsafe positive promotion');
+
+  console.log(`\nResults: ${passed} passed, ${failed} failed\n`);
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+module.exports = {
+  validateFeedbackMemory,
+  resolveFeedbackAction,
+  prepareForStorage,
+  parseTimestamp,
+  GENERIC_TAGS,
+  MIN_CONTENT_LENGTH,
+  VALID_TITLE_PREFIXES,
+  VALID_CATEGORIES,
+};
+
+if (require.main === module && process.argv.includes('--test')) {
+  runTests();
+}

package/scripts/feedback-to-memory.js

@@ -0,0 +1,165 @@
+#!/usr/bin/env node
+/**
+ * Feedback → Memory Bridge
+ *
+ * Converts raw feedback params into a schema-validated memory object
+ * suitable for mcp__memory__remember. This is the validation boundary
+ * between external feedback signals and MCP memory storage.
+ *
+ * Usage:
+ *   echo '{"signal":"negative","context":"...","whatWentWrong":"...","tags":["testing"]}' | node scripts/feedback-to-memory.js
+ *   node scripts/feedback-to-memory.js --test
+ *
+ * Input (stdin JSON):
+ *   signal: "positive" | "negative"
+ *   context: string — what the agent was doing
+ *   whatWentWrong: string — for negative: what failed
+ *   whatToChange: string — for negative: how to avoid
+ *   whatWorked: string — for positive: the pattern to repeat
+ *   tags: string[] — domain tags (at least 1 non-generic)
+ *
+ * Output (stdout JSON):
+ *   { ok: true, memory: { title, content, category, importance, tags } }
+ *   { ok: false, reason: string, issues?: string[] }
+ */
+'use strict';
+
+const { resolveFeedbackAction, prepareForStorage } = require('./feedback-schema');
+
+function convertFeedbackToMemory(params) {
+  const action = resolveFeedbackAction({
+    signal: params.signal,
+    context: params.context || '',
+    whatWentWrong: params.whatWentWrong,
+    whatToChange: params.whatToChange,
+    whatWorked: params.whatWorked,
+    tags: params.tags || [],
+  });
+
+  if (!action || action.type === 'no-action') {
+    return { ok: false, reason: action ? action.reason : 'Unknown action resolution failure' };
+  }
+
+  const prep = prepareForStorage(action.memory);
+  if (!prep.ok) {
+    return { ok: false, reason: `Schema validation failed: ${prep.issues.join('; ')}`, issues: prep.issues };
+  }
+
+  return { ok: true, actionType: action.type, memory: prep.memory };
+}
+
+// ---------------------------------------------------------------------------
+// CLI: stdin mode
+// ---------------------------------------------------------------------------
+
+function runStdin() {
+  let input = '';
+  process.stdin.setEncoding('utf-8');
+  process.stdin.on('data', (chunk) => { input += chunk; });
+  process.stdin.on('end', () => {
+    try {
+      const params = JSON.parse(input.trim());
+      const result = convertFeedbackToMemory(params);
+      process.stdout.write(JSON.stringify(result, null, 2) + '\n');
+      process.exit(result.ok ? 0 : 2);
+    } catch (err) {
+      process.stdout.write(JSON.stringify({ ok: false, reason: `Parse error: ${err.message}` }, null, 2) + '\n');
+      process.exit(1);
+    }
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Built-in Tests
+// ---------------------------------------------------------------------------
+
+function runTests() {
+  let passed = 0;
+  let failed = 0;
+
+  function assert(condition, name) {
+    if (condition) {
+      passed++;
+      console.log(`  ✅ ${name}`);
+    } else {
+      failed++;
+      console.log(`  ❌ ${name}`);
+    }
+  }
+
+  console.log('\n🧪 feedback-to-memory.js — Tests\n');
+
+  // Valid negative feedback → memory
+  const neg = convertFeedbackToMemory({
+    signal: 'negative',
+    context: 'Agent claimed fix without test evidence',
+    whatWentWrong: 'No tests were run before claiming the bug was fixed',
+    whatToChange: 'Always run tests and show output before claiming done',
+    tags: ['verification', 'testing'],
+  });
+  assert(neg.ok === true, 'valid negative → ok');
+  assert(neg.actionType === 'store-mistake', 'negative → store-mistake');
+  assert(neg.memory.title.startsWith('MISTAKE:'), 'negative → MISTAKE: prefix');
+  assert(neg.memory.category === 'error', 'negative → error category');
+  assert(neg.memory.tags.includes('verification'), 'preserves domain tags');
+
+  // Valid positive feedback → memory
+  const pos = convertFeedbackToMemory({
+    signal: 'positive',
+    whatWorked: 'Built schema-validated feedback system with prevention rules',
+    tags: ['architecture', 'rlhf'],
+  });
+  assert(pos.ok === true, 'valid positive → ok');
+  assert(pos.actionType === 'store-learning', 'positive → store-learning');
+  assert(pos.memory.title.startsWith('SUCCESS:'), 'positive → SUCCESS: prefix');
+  assert(pos.memory.category === 'learning', 'positive → learning category');
+
+  // Bare thumbs down → rejected
+  const bare = convertFeedbackToMemory({ signal: 'negative' });
+  assert(bare.ok === false, 'bare negative → rejected');
+  assert(bare.reason.includes('No context') || bare.reason.includes('cannot'), 'reports missing context');
+
+  // Bare thumbs up → rejected
+  const bareUp = convertFeedbackToMemory({ signal: 'positive' });
+  assert(bareUp.ok === false, 'bare positive → rejected');
+
+  // Unknown signal → rejected
+  const unknown = convertFeedbackToMemory({ signal: 'maybe', context: 'test' });
+  assert(unknown.ok === false, 'unknown signal → rejected');
+
+  // Context-only negative → ok
+  const ctxNeg = convertFeedbackToMemory({
+    signal: 'negative',
+    context: 'Showed fake RLHF statistics panel to user',
+    tags: ['rlhf'],
+  });
+  assert(ctxNeg.ok === true, 'context-only negative → ok');
+
+  // Context-only positive → ok
+  const ctxPos = convertFeedbackToMemory({
+    signal: 'positive',
+    context: 'Ran full test suite and showed green output before responding',
+    tags: ['verification'],
+  });
+  assert(ctxPos.ok === true, 'context-only positive → ok');
+
+  console.log(`\n${'═'.repeat(50)}`);
+  console.log(`Results: ${passed} passed, ${failed} failed, ${passed + failed} total`);
+  console.log(`${'═'.repeat(50)}\n`);
+
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+// ---------------------------------------------------------------------------
+// Exports & main
+// ---------------------------------------------------------------------------
+
+module.exports = { convertFeedbackToMemory };
+
+if (require.main === module) {
+  if (process.argv.includes('--test')) {
+    runTests();
+  } else {
+    runStdin();
+  }
+}

package/scripts/feedback-to-rules.js

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_LOG = path.join(__dirname, '..', '.claude', 'memory', 'feedback', 'feedback-log.jsonl');
+const NEG = new Set(['negative', 'negative_strong', 'down', 'thumbs_down']);
+const POS = new Set(['positive', 'positive_strong', 'up', 'thumbs_up']);
+
+function parseFeedbackFile(filePath) {
+  if (!fs.existsSync(filePath)) return [];
+  const entries = [];
+  for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try { entries.push(JSON.parse(trimmed)); } catch { /* skip malformed */ }
+  }
+  return entries;
+}
+
+function classifySignal(entry) {
+  const sig = (entry.signal || entry.feedback || '').toLowerCase();
+  if (NEG.has(sig)) return 'negative';
+  if (POS.has(sig)) return 'positive';
+  return null;
+}
+
+function normalize(ctx) {
+  return (ctx || '').replace(/\/Users\/[^\s/]+/g, '~').replace(/:[0-9]+/g, '').toLowerCase().trim();
+}
+
+function analyze(entries) {
+  let positiveCount = 0, negativeCount = 0;
+  const categories = {};
+  const toolBuckets = {};
+  const contextCounts = {};
+
+  for (const e of entries) {
+    const cls = classifySignal(e);
+    if (!cls) continue;
+    cls === 'positive' ? positiveCount++ : negativeCount++;
+
+    const cat = e.task_category || e.category || 'uncategorized';
+    categories[cat] = categories[cat] || { positive: 0, negative: 0, total: 0 };
+    categories[cat][cls]++;
+    categories[cat].total++;
+
+    if (cls === 'negative') {
+      const tool = e.tool_name || 'unknown';
+      toolBuckets[tool] = (toolBuckets[tool] || 0) + 1;
+      const key = normalize(e.context);
+      if (key.length > 10) {
+        if (!contextCounts[key]) contextCounts[key] = { raw: e.context, count: 0, tool };
+        contextCounts[key].count++;
+      }
+    }
+  }
+
+  const total = positiveCount + negativeCount;
+  const recurringIssues = Object.values(contextCounts)
+    .filter(v => v.count >= 2)
+    .sort((a, b) => b.count - a.count)
+    .map(v => ({
+      pattern: v.raw.slice(0, 120),
+      count: v.count,
+      severity: v.count >= 4 ? 'critical' : v.count >= 3 ? 'high' : 'medium',
+      suggestedRule: `NEVER ${v.raw.slice(0, 80).replace(/CRITICAL ERROR - User frustrated: /i, '')}`,
+    }));
+
+  return {
+    generatedAt: new Date().toISOString(),
+    totalFeedback: total,
+    negativeCount,
+    positiveCount,
+    negativeRate: total ? `${((negativeCount / total) * 100).toFixed(1)}%` : '0%',
+    recurringIssues,
+    categoryBreakdown: categories,
+    topTools: toolBuckets,
+  };
+}
+
+function toRules(report) {
+  const lines = ['# Suggested Rules from Feedback Analysis', `# Generated: ${report.generatedAt}`, ''];
+  lines.push(`# Negative rate: ${report.negativeRate} (${report.negativeCount}/${report.totalFeedback})`);
+  lines.push('');
+  for (const issue of report.recurringIssues) {
+    lines.push(`- [${issue.severity.toUpperCase()}] (${issue.count}x) ${issue.suggestedRule}`);
+  }
+  if (!report.recurringIssues.length) lines.push('- No recurring issues detected.');
+  return lines.join('\n');
+}
+
+if (require.main === module) {
+  try {
+    const logPath = process.argv[2] && !process.argv[2].startsWith('--') ? process.argv[2] : DEFAULT_LOG;
+    const entries = parseFeedbackFile(logPath);
+    const report = analyze(entries);
+    if (process.argv.includes('--rules')) {
+      console.log(toRules(report));
+    } else {
+      console.log(JSON.stringify(report, null, 2));
+    }
+  } catch (err) {
+    console.error('Warning:', err.message);
+  }
+  process.exit(0);
+}
+
+module.exports = { parseFeedbackFile, classifySignal, analyze, toRules, normalize };

package/scripts/generate-paperbanana-diagrams.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT_DIR"
+
+if [[ -f .env.paperbanana ]]; then
+  # shellcheck source=/dev/null
+  source .env.paperbanana
+fi
+
+if [[ ! -f .env ]]; then
+  echo "Missing .env. Add GEMINI_API_KEY first."
+  exit 1
+fi
+
+# shellcheck source=/dev/null
+source .env
+
+if [[ -z "${GOOGLE_API_KEY:-}" && -z "${GEMINI_API_KEY:-}" ]]; then
+  echo "GEMINI_API_KEY or GOOGLE_API_KEY is required in .env"
+  exit 1
+fi
+
+# Prefer explicit GOOGLE_API_KEY since paperbanana/google-genai prioritizes it.
+if [[ -z "${GOOGLE_API_KEY:-}" ]]; then
+  export GOOGLE_API_KEY="$GEMINI_API_KEY"
+fi
+
+PB_ESTIMATE_PER_DIAGRAM="${PB_ESTIMATE_PER_DIAGRAM:-0.80}"
+PB_MAX_ITERATIONS="${PB_MAX_ITERATIONS:-1}"
+PB_VLM_MODEL="${PB_VLM_MODEL:-gemini-2.5-flash}"
+PB_IMAGE_MODEL="${PB_IMAGE_MODEL:-gemini-3-pro-image-preview}"
+
+budget_check() {
+  local estimate="$1"
+  node -e '
+    const { getBudgetStatus } = require("./scripts/budget-guard");
+    const est = Number(process.argv[1]);
+    const s = getBudgetStatus();
+    const projected = s.totalUsd + est;
+    if (projected > s.budgetUsd) {
+      console.error(`Blocked: projected spend ${projected.toFixed(2)} exceeds budget ${s.budgetUsd.toFixed(2)} USD`);
+      process.exit(3);
+    }
+  ' "$estimate"
+}
+
+mkdir -p docs/diagrams
+
+cat > docs/diagrams/rlhf-architecture.txt <<'TXT'
+The system starts when a user gives explicit thumbs up/down feedback. A capture layer enriches the signal with context and tags.
+An action resolver maps the signal to store-learning, store-mistake, or no-action.
+A schema validator enforces strict structure before memory promotion.
+Valid records are stored in a local memory log categorized as error or learning.
+An analytics layer computes quality trends and recurrence patterns.
+A prevention-rule engine converts repeated mistakes into hard guardrails.
+A DPO export layer pairs learning and error memories into prompt/chosen/rejected JSONL.
+All channels (ChatGPT Actions, Claude MCP, Codex MCP, Gemini tools, Amp skills) route through one shared API and policy core.
+TXT
+
+cat > docs/diagrams/plugin-topology.txt <<'TXT'
+Show a central RLHF Feedback API with five adapters around it.
+Adapter 1: ChatGPT via GPT Actions OpenAPI.
+Adapter 2: Claude via local MCP server and .mcp.json.
+Adapter 3: Codex via MCP server config.toml.
+Adapter 4: Gemini via function-calling tool declarations.
+Adapter 5: Amp via skills template.
+Include bidirectional arrows from each adapter to the API.
+Under the API place three internal modules: schema validation, prevention rules, and DPO export.
+Add a budget guard module enforcing a strict monthly cost cap of 10 USD.
+TXT
+
+budget_check "$PB_ESTIMATE_PER_DIAGRAM"
+paperbanana generate \
+  --input docs/diagrams/rlhf-architecture.txt \
+  --caption "RLHF Feedback Loop architecture for AI coding agents with schema gate, memory store, prevention rules, and DPO export" \
+  --vlm-provider gemini \
+  --vlm-model "$PB_VLM_MODEL" \
+  --image-provider google_imagen \
+  --image-model "$PB_IMAGE_MODEL" \
+  --iterations "$PB_MAX_ITERATIONS" \
+  --output docs/diagrams/rlhf-architecture.png
+node scripts/budget-guard.js --add="$PB_ESTIMATE_PER_DIAGRAM" --source=paperbanana --note="architecture-overview"
+
+budget_check "$PB_ESTIMATE_PER_DIAGRAM"
+paperbanana generate \
+  --input docs/diagrams/plugin-topology.txt \
+  --caption "Go-to-market plugin topology: ChatGPT Actions, Claude MCP, Codex MCP, Gemini function calling, and Amp skills through one RLHF API core" \
+  --vlm-provider gemini \
+  --vlm-model "$PB_VLM_MODEL" \
+  --image-provider google_imagen \
+  --image-model "$PB_IMAGE_MODEL" \
+  --iterations "$PB_MAX_ITERATIONS" \
+  --output docs/diagrams/plugin-topology.png
+node scripts/budget-guard.js --add="$PB_ESTIMATE_PER_DIAGRAM" --source=paperbanana --note="plugin-topology"
+
+echo "Generated diagrams:"
+ls -la docs/diagrams/*.png