rlhf-feedback-loop 0.5.0

package/scripts/feedback-attribution.js

@@ -0,0 +1,313 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// rlhf: scripts/ is 1 level below repo root (not 2 like Subway .claude/scripts/feedback/)
+const ROOT = path.join(__dirname, '..');
+const PATHS = {
+  actionLog: path.join(ROOT, '.claude', 'memory', 'feedback', 'action-log.jsonl'),
+  attributions: path.join(ROOT, '.claude', 'memory', 'feedback', 'feedback-attributions.jsonl'),
+  attributedFeedback: path.join(ROOT, '.claude', 'memory', 'feedback', 'attributed-feedback.jsonl'),
+};
+
+const STOPWORDS = new Set([
+  'about', 'after', 'again', 'allow', 'already', 'always', 'because', 'before', 'being', 'between',
+  'could', 'does', 'done', 'each', 'ensure', 'every', 'from', 'have', 'into', 'just', 'make', 'more',
+  'most', 'must', 'never', 'only', 'other', 'over', 'repeat', 'same', 'should', 'since', 'that',
+  'their', 'them', 'then', 'there', 'these', 'they', 'this', 'under', 'until', 'very', 'what', 'when',
+  'where', 'which', 'while', 'with', 'without', 'would', 'thumbs', 'down', 'up', 'please', 'avoid',
+]);
+
+function readJsonl(filePath, maxLines = 500) {
+  if (!filePath || !fs.existsSync(filePath)) return [];
+  const lines = fs.readFileSync(filePath, 'utf8').split('\n');
+  const out = [];
+  for (let i = lines.length - 1; i >= 0 && out.length < maxLines; i -= 1) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    try {
+      out.push(JSON.parse(line));
+    } catch {
+      // ignore malformed jsonl lines
+    }
+  }
+  return out.reverse();
+}
+
+function appendJsonl(filePath, obj) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.appendFileSync(filePath, `${JSON.stringify(obj)}\n`);
+}
+
+function normalize(text) {
+  return String(text || '')
+    .replace(/\/Users\/[^\s/]+/g, '~')
+    .replace(/:[0-9]+/g, '')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .toLowerCase();
+}
+
+function stripFeedbackPrefix(text) {
+  return String(text || '')
+    .replace(/^\s*thumbs?[ -]?down\s*[:\-]?\s*/i, '')
+    .replace(/^\s*thumbs?[ -]?up\s*[:\-]?\s*/i, '')
+    .replace(/^\s*(negative|positive)\s+feedback\s*[:\-]?\s*/i, '')
+    .trim();
+}
+
+function tokenize(text) {
+  return [...new Set(
+    normalize(text)
+      .split(/[^a-z0-9]+/)
+      .filter(Boolean)
+      .filter(w => w.length >= 3 && !STOPWORDS.has(w))
+  )].slice(0, 24);
+}
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function hashText(text) {
+  let h = 2166136261;
+  const str = String(text || '');
+  for (let i = 0; i < str.length; i += 1) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return (h >>> 0).toString(16).padStart(8, '0');
+}
+
+function summarizeToolInput(toolName, toolInput) {
+  const tool = String(toolName || 'unknown');
+  const input = (typeof toolInput === 'string') ? toolInput : JSON.stringify(toolInput || {});
+  if (tool === 'Bash') {
+    try {
+      const parsed = typeof toolInput === 'string' ? JSON.parse(toolInput) : toolInput;
+      if (parsed && typeof parsed.command === 'string') {
+        return parsed.command;
+      }
+    } catch {}
+    return input;
+  }
+  if (tool === 'Edit' || tool === 'Write') {
+    try {
+      const parsed = typeof toolInput === 'string' ? JSON.parse(toolInput) : toolInput;
+      const p = parsed?.file_path || parsed?.path || parsed?.filePath || '';
+      if (p) return `${tool} ${p}`;
+    } catch {}
+  }
+  return input;
+}
+
+function inferIntent(toolName, text) {
+  const t = normalize(text);
+  if (/git\s+push|--force|git\s+reset|git\s+checkout/.test(t)) return 'git-risk';
+  if (/rm\s+-rf|sudo|chmod\s+777|chown\s+-r/.test(t)) return 'destructive-shell';
+  if (/\.env|secret|token|credential/.test(t)) return 'sensitive-data';
+  if (toolName === 'Edit' || toolName === 'Write') return 'file-change';
+  if (toolName === 'Bash') return 'shell-command';
+  return 'general';
+}
+
+function riskScore(toolName, text) {
+  const t = normalize(text);
+  let score = 2;
+  if (toolName === 'Bash') score += 2;
+  if (toolName === 'Edit' || toolName === 'Write') score += 1;
+  if (/--force|rm\s+-rf|reset\s+--hard|chmod\s+777|sudo/.test(t)) score += 4;
+  if (/\.env|secret|token|credential/.test(t)) score += 3;
+  return Math.max(1, Math.min(10, score));
+}
+
+function overlapScore(tokensA, tokensB) {
+  const a = new Set(tokensA || []);
+  const b = new Set(tokensB || []);
+  if (a.size === 0 || b.size === 0) return 0;
+  let inter = 0;
+  for (const w of a) {
+    if (b.has(w)) inter += 1;
+  }
+  const union = a.size + b.size - inter;
+  return union > 0 ? inter / union : 0;
+}
+
+function scoreCandidate(action, feedbackNormalized, feedbackTokens, nowMs) {
+  const ts = Date.parse(action.timestamp || '') || nowMs;
+  const ageMs = Math.max(0, nowMs - ts);
+  const recency = Math.exp(-ageMs / (6 * 60 * 1000));
+  const actionTokens = Array.isArray(action.keywords) && action.keywords.length > 0
+    ? action.keywords
+    : tokenize(action.normalized_input || action.input || '');
+  const lexical = overlapScore(feedbackTokens, actionTokens);
+  const containment = feedbackNormalized && normalize(action.normalized_input || '').includes(feedbackNormalized) ? 1 : 0;
+  const risk = Number(action.risk_score || 0) / 10;
+  let score = 0.45 * lexical + 0.30 * recency + 0.20 * risk + 0.05 * containment;
+
+  const f = feedbackNormalized;
+  if (/\bgit\b|\bpush\b|\bforce\b/.test(f) && action.tool_name === 'Bash') score += 0.1;
+  if (/\bedit\b|\bwrite\b|\bfile\b/.test(f) && (action.tool_name === 'Edit' || action.tool_name === 'Write')) score += 0.1;
+
+  return Math.max(0, Math.min(1, score));
+}
+
+function recordAction(toolName, toolInput, opts = {}) {
+  const actionLogPath = opts.actionLogPath || process.env.RLHF_ACTION_LOG || PATHS.actionLog;
+  const tool = String(toolName || 'unknown');
+  const inputSummary = summarizeToolInput(tool, toolInput);
+  const normalized = normalize(inputSummary);
+  const row = {
+    action_id: `act_${Date.now()}_${hashText(`${tool}|${normalized}`)}`,
+    timestamp: nowIso(),
+    tool_name: tool,
+    input: inputSummary,
+    normalized_input: normalized,
+    intent: inferIntent(tool, inputSummary),
+    keywords: tokenize(inputSummary),
+    risk_score: riskScore(tool, inputSummary),
+  };
+  appendJsonl(actionLogPath, row);
+  return { ok: true, action: row, actionLogPath };
+}
+
+function attributeFeedback(signal, feedbackContext, opts = {}) {
+  const sig = String(signal || '').toLowerCase().trim();
+  const actionLogPath = opts.actionLogPath || process.env.RLHF_ACTION_LOG || PATHS.actionLog;
+  const attributionsPath = opts.attributionsPath || process.env.RLHF_FEEDBACK_ATTRIBUTIONS || PATHS.attributions;
+  const attributedFeedbackPath = opts.attributedFeedbackPath || process.env.RLHF_ATTRIBUTED_FEEDBACK || PATHS.attributedFeedback;
+
+  if (sig !== 'negative' && sig !== 'positive') {
+    return { ok: true, skipped: true, reason: 'signal_not_supported' };
+  }
+
+  const nowMs = Date.now();
+  const feedbackText = stripFeedbackPrefix(feedbackContext);
+  const feedbackNormalized = normalize(feedbackText);
+  const feedbackTokens = tokenize(feedbackText);
+  const windowMs = Number(opts.windowMs || process.env.RLHF_ATTRIBUTION_WINDOW_MS || (20 * 60 * 1000));
+  const maxActions = Number(opts.maxActions || 80);
+  const topK = Number(opts.topK || 3);
+  const threshold = Number(opts.threshold || (sig === 'negative' ? 0.33 : 0.38));
+
+  const actions = readJsonl(actionLogPath, maxActions)
+    .filter(a => {
+      const ts = Date.parse(a.timestamp || '') || nowMs;
+      return (nowMs - ts) <= windowMs;
+    })
+    .reverse();
+
+  const scored = actions.map(action => {
+    const confidence = scoreCandidate(action, feedbackNormalized, feedbackTokens, nowMs);
+    return { action, confidence };
+  }).sort((a, b) => b.confidence - a.confidence);
+
+  const selected = scored.filter(s => s.confidence >= threshold).slice(0, topK);
+
+  const attribution = {
+    attribution_id: `att_${Date.now()}_${hashText(`${sig}|${feedbackNormalized}`)}`,
+    timestamp: nowIso(),
+    signal: sig,
+    feedback_context: feedbackContext,
+    feedback_normalized: feedbackNormalized,
+    threshold,
+    candidates_considered: scored.length,
+    attributed_actions: selected.map(s => ({
+      action_id: s.action.action_id,
+      tool_name: s.action.tool_name,
+      input: s.action.input,
+      normalized_input: s.action.normalized_input,
+      intent: s.action.intent,
+      confidence: Number(s.confidence.toFixed(4)),
+      risk_score: s.action.risk_score,
+    })),
+  };
+
+  appendJsonl(attributionsPath, attribution);
+
+  if (sig === 'negative') {
+    for (const s of selected) {
+      appendJsonl(attributedFeedbackPath, {
+        timestamp: attribution.timestamp,
+        signal: sig,
+        feedback: sig,
+        source: 'attributed',
+        source_detail: 'feedback-attribution-engine',
+        context: s.action.input,
+        tool_name: s.action.tool_name,
+        attributed_action_id: s.action.action_id,
+        attribution_id: attribution.attribution_id,
+        confidence: Number(s.confidence.toFixed(4)),
+        intent: s.action.intent,
+      });
+    }
+  }
+
+  return {
+    ok: true,
+    signal: sig,
+    attributedCount: selected.length,
+    topConfidence: selected.length ? Number(selected[0].confidence.toFixed(4)) : 0,
+    attributionId: attribution.attribution_id,
+    actionLogPath,
+    attributionsPath,
+    attributedFeedbackPath,
+  };
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const recordIdx = args.indexOf('--record-action');
+  const attrIdx = args.indexOf('--attribute');
+
+  if (recordIdx >= 0) {
+    const toolName = args[recordIdx + 1] || 'unknown';
+    const toolInput = args[recordIdx + 2] || '{}';
+    const result = recordAction(toolName, toolInput);
+    process.stdout.write(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  if (attrIdx >= 0) {
+    const signal = args[attrIdx + 1] || '';
+    const context = args[attrIdx + 2] || '';
+    const result = attributeFeedback(signal, context);
+    process.stdout.write(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  process.stdout.write(JSON.stringify({
+    ok: true,
+    usage: [
+      '--record-action <toolName> <toolInputJson>',
+      '--attribute <signal> <feedbackContext>',
+    ],
+  }, null, 2));
+}
+
+if (require.main === module) {
+  try {
+    main();
+  } catch (error) {
+    console.error(`[ATTRIBUTION WARNING] ${error.message}`);
+    process.exit(0);
+  }
+}
+
+module.exports = {
+  PATHS,
+  readJsonl,
+  appendJsonl,
+  normalize,
+  stripFeedbackPrefix,
+  tokenize,
+  inferIntent,
+  riskScore,
+  summarizeToolInput,
+  overlapScore,
+  scoreCandidate,
+  recordAction,
+  attributeFeedback,
+};

package/scripts/feedback-inbox-read.js

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+/**
+ * Feedback Inbox Reader
+ *
+ * Reads new feedback entries from the inbox JSONL file, using a cursor
+ * to avoid reprocessing. External systems (Phoenix bridge, other agents)
+ * append feedback signals to the inbox; Amp's reflexion-preflight skill
+ * calls this script each turn to ingest new signals.
+ *
+ * Usage:
+ *   node scripts/feedback-inbox-read.js              # output new entries as JSON array
+ *   node scripts/feedback-inbox-read.js --peek        # show count without advancing cursor
+ *   node scripts/feedback-inbox-read.js --reset       # reset cursor to re-read all
+ *   node scripts/feedback-inbox-read.js --test        # run built-in tests
+ *
+ * Inbox:  .claude/feedback-loop/inbox.jsonl
+ * Cursor: .claude/feedback-loop/inbox.cursor.json
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const INBOX_PATH = path.join(PROJECT_ROOT, '.claude', 'feedback-loop', 'inbox.jsonl');
+const CURSOR_PATH = path.join(PROJECT_ROOT, '.claude', 'feedback-loop', 'inbox.cursor.json');
+
+function readInbox() {
+  if (!fs.existsSync(INBOX_PATH)) return [];
+  const raw = fs.readFileSync(INBOX_PATH, 'utf-8').trim();
+  if (!raw) return [];
+  return raw.split('\n').map((line, idx) => {
+    try {
+      return { _lineIndex: idx, ...JSON.parse(line) };
+    } catch {
+      return null;
+    }
+  }).filter(Boolean);
+}
+
+function loadCursor() {
+  if (!fs.existsSync(CURSOR_PATH)) return { lastLineIndex: -1 };
+  try {
+    return JSON.parse(fs.readFileSync(CURSOR_PATH, 'utf-8'));
+  } catch {
+    return { lastLineIndex: -1 };
+  }
+}
+
+function saveCursor(cursor) {
+  const dir = path.dirname(CURSOR_PATH);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(CURSOR_PATH, JSON.stringify(cursor, null, 2) + '\n');
+}
+
+function getNewEntries(advance) {
+  const entries = readInbox();
+  if (entries.length === 0) return [];
+
+  const cursor = loadCursor();
+  const newEntries = entries.filter((e) => e._lineIndex > cursor.lastLineIndex);
+
+  if (advance && newEntries.length > 0) {
+    const maxIdx = Math.max(...newEntries.map((e) => e._lineIndex));
+    saveCursor({ lastLineIndex: maxIdx, updatedAt: new Date().toISOString() });
+  }
+
+  // Strip internal _lineIndex before returning
+  return newEntries.map(({ _lineIndex, ...rest }) => rest);
+}
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+function runTests() {
+  const os = require('os');
+  let passed = 0;
+  let failed = 0;
+
+  function assert(condition, name) {
+    if (condition) { passed++; console.log(`  ✅ ${name}`); }
+    else { failed++; console.log(`  ❌ ${name}`); }
+  }
+
+  console.log('\n🧪 feedback-inbox-read.js — Tests\n');
+
+  // Setup temp inbox
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'inbox-test-'));
+  const tmpInbox = path.join(tmpDir, 'inbox.jsonl');
+  const tmpCursor = path.join(tmpDir, 'inbox.cursor.json');
+
+  // Write test entries
+  const entries = [
+    { signal: 'negative', context: 'Bad thing happened', tags: ['testing'] },
+    { signal: 'positive', context: 'Good thing happened', tags: ['testing'] },
+    { signal: 'negative', context: 'Another bad thing', tags: ['rlhf'] },
+  ];
+  fs.writeFileSync(tmpInbox, entries.map((e) => JSON.stringify(e)).join('\n') + '\n');
+
+  // Test reading all (no cursor)
+  const allEntries = (() => {
+    const raw = fs.readFileSync(tmpInbox, 'utf-8').trim();
+    return raw.split('\n').map((line, idx) => {
+      try { return { _lineIndex: idx, ...JSON.parse(line) }; }
+      catch { return null; }
+    }).filter(Boolean);
+  })();
+  assert(allEntries.length === 3, 'reads all 3 entries from inbox');
+
+  // Test cursor-based filtering
+  fs.writeFileSync(tmpCursor, JSON.stringify({ lastLineIndex: 0 }));
+  const afterFirst = allEntries.filter((e) => e._lineIndex > 0);
+  assert(afterFirst.length === 2, 'cursor at 0 → 2 new entries');
+
+  fs.writeFileSync(tmpCursor, JSON.stringify({ lastLineIndex: 2 }));
+  const afterAll = allEntries.filter((e) => e._lineIndex > 2);
+  assert(afterAll.length === 0, 'cursor at 2 → 0 new entries');
+
+  // Test loadCursor with missing file
+  const missingCursor = path.join(tmpDir, 'missing.cursor.json');
+  const fakeMod = { loadCursor: () => {
+    if (!fs.existsSync(missingCursor)) return { lastLineIndex: -1 };
+    try { return JSON.parse(fs.readFileSync(missingCursor, 'utf-8')); } catch { return { lastLineIndex: -1 }; }
+  }};
+  assert(fakeMod.loadCursor().lastLineIndex === -1, 'loadCursor returns -1 for missing file');
+
+  // Test saveCursor creates directory
+  const deepDir = path.join(tmpDir, 'deep', 'nested');
+  const deepCursor = path.join(deepDir, 'cursor.json');
+  if (!fs.existsSync(deepDir)) fs.mkdirSync(deepDir, { recursive: true });
+  fs.writeFileSync(deepCursor, JSON.stringify({ lastLineIndex: 5, updatedAt: new Date().toISOString() }) + '\n');
+  const loaded = JSON.parse(fs.readFileSync(deepCursor, 'utf-8'));
+  assert(loaded.lastLineIndex === 5, 'saveCursor persists cursor value');
+
+  // Cleanup
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+
+  console.log(`\n${'═'.repeat(50)}`);
+  console.log(`Results: ${passed} passed, ${failed} failed, ${passed + failed} total`);
+  console.log(`${'═'.repeat(50)}\n`);
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+if (require.main === module) {
+  if (process.argv.includes('--test')) {
+    runTests();
+  } else if (process.argv.includes('--reset')) {
+    if (fs.existsSync(CURSOR_PATH)) fs.unlinkSync(CURSOR_PATH);
+    console.log('Cursor reset.');
+  } else {
+    const peek = process.argv.includes('--peek');
+    const entries = getNewEntries(!peek);
+    if (entries.length === 0) {
+      // Silent — no new entries
+      process.exit(0);
+    }
+    process.stdout.write(JSON.stringify(entries, null, 2) + '\n');
+  }
+}
+
+module.exports = { getNewEntries, readInbox, loadCursor, saveCursor, INBOX_PATH, CURSOR_PATH };