rlhf-feedback-loop 0.5.0

package/scripts/validate-feedback.js

@@ -0,0 +1,580 @@
+'use strict';
+/**
+ * Feedback Data Quality Validator
+ *
+ * Implements a 4-level validation pipeline:
+ * 1. Schema validation (required fields, value ranges)
+ * 2. Semantic validation (logical consistency)
+ * 3. Anomaly detection (suspicious patterns, sensitive data)
+ * 4. Self-correction (auto-correct fixable errors)
+ *
+ * Ported from Subway_RN_Demo with rlhf schema adaptations:
+ * - Uses 'signal' (not 'feedback') with values 'positive'/'negative'
+ * - Uses 'id' as required field (not 'source')
+ * - RLHF_FEEDBACK_DIR env var for path resolution
+ *
+ * Usage (CLI):
+ *   echo '{"signal":"positive",...}' | node validate-feedback.js
+ *   node validate-feedback.js --audit   # Audit existing feedback log
+ *   node validate-feedback.js --stats   # Show quality statistics
+ *
+ * Usage (module):
+ *   const { validateEntry } = require('./validate-feedback');
+ *
+ * LOCAL ONLY - Do not commit feedback log data to repository
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+// =============================================================================
+// PATH RESOLUTION
+// =============================================================================
+
+const DEFAULT_FEEDBACK_DIR = path.join(__dirname, '..', '.claude', 'memory', 'feedback');
+
+function getFeedbackDir() {
+  return process.env.RLHF_FEEDBACK_DIR || DEFAULT_FEEDBACK_DIR;
+}
+
+function getFeedbackPaths() {
+  const dir = getFeedbackDir();
+  return {
+    FEEDBACK_LOG: path.join(dir, 'feedback-log.jsonl'),
+    VALIDATION_LOG: path.join(dir, 'validation-issues.jsonl'),
+    QUALITY_REPORT: path.join(dir, 'quality-report.json'),
+  };
+}
+
+// =============================================================================
+// SCHEMA VALIDATION (Level 1)
+// =============================================================================
+
+const REQUIRED_FIELDS = ['timestamp', 'signal', 'id'];
+const VALID_SIGNAL_VALUES = ['positive', 'negative'];
+const VALID_REWARD_RANGE = [-1, 1];
+
+function validateSchema(entry) {
+  const issues = [];
+
+  // Check required fields
+  for (const field of REQUIRED_FIELDS) {
+    if (!(field in entry)) {
+      issues.push({
+        level: 'error',
+        field,
+        message: `Missing required field: ${field}`,
+        suggestion: `Add "${field}" to the feedback entry`,
+      });
+    }
+  }
+
+  // Validate signal value
+  if (entry.signal && !VALID_SIGNAL_VALUES.includes(entry.signal)) {
+    issues.push({
+      level: 'warning',
+      field: 'signal',
+      message: `Invalid signal value: "${entry.signal}"`,
+      suggestion: `Use one of: ${VALID_SIGNAL_VALUES.join(', ')}`,
+    });
+  }
+
+  // Validate reward range
+  if ('reward' in entry) {
+    if (
+      typeof entry.reward !== 'number' ||
+      entry.reward < VALID_REWARD_RANGE[0] ||
+      entry.reward > VALID_REWARD_RANGE[1]
+    ) {
+      issues.push({
+        level: 'error',
+        field: 'reward',
+        message: `Reward out of range: ${entry.reward}`,
+        suggestion: `Reward must be between ${VALID_REWARD_RANGE[0]} and ${VALID_REWARD_RANGE[1]}`,
+      });
+    }
+  }
+
+  // Validate timestamp format
+  if (entry.timestamp) {
+    const ts = new Date(entry.timestamp);
+    if (isNaN(ts.getTime())) {
+      issues.push({
+        level: 'error',
+        field: 'timestamp',
+        message: `Invalid timestamp format: "${entry.timestamp}"`,
+        suggestion: 'Use ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ',
+      });
+    } else if (ts > new Date()) {
+      issues.push({
+        level: 'warning',
+        field: 'timestamp',
+        message: 'Timestamp is in the future',
+        suggestion: 'Check system clock synchronization',
+      });
+    }
+  }
+
+  return issues;
+}
+
+// =============================================================================
+// SEMANTIC VALIDATION (Level 2)
+// =============================================================================
+
+function validateSemantics(entry) {
+  const issues = [];
+
+  // Signal-reward consistency
+  if (entry.signal === 'positive' && typeof entry.reward === 'number' && entry.reward < 0) {
+    issues.push({
+      level: 'error',
+      field: 'reward',
+      message: 'Positive signal but negative reward',
+      explanation: 'Semantic inconsistency: positive signal should have reward >= 0',
+      suggestion: 'Either change signal to "negative" or reward to positive value',
+    });
+  }
+
+  if (entry.signal === 'negative' && typeof entry.reward === 'number' && entry.reward > 0) {
+    issues.push({
+      level: 'error',
+      field: 'reward',
+      message: 'Negative signal but positive reward',
+      explanation: 'Semantic inconsistency: negative signal should have reward <= 0',
+      suggestion: 'Either change signal to "positive" or reward to negative value',
+    });
+  }
+
+  // Context validation
+  if (entry.context !== undefined) {
+    // Empty or too short context
+    if (typeof entry.context === 'string' && entry.context.trim().length < 5) {
+      issues.push({
+        level: 'warning',
+        field: 'context',
+        message: 'Context too short to be meaningful',
+        explanation: 'Short context reduces ML training value',
+        suggestion: 'Provide more descriptive context (at least 10 characters)',
+      });
+    }
+
+    // Check for placeholder text
+    const placeholders = ['TODO', 'FIXME', 'placeholder', 'test', 'example'];
+    for (const ph of placeholders) {
+      if (
+        typeof entry.context === 'string' &&
+        entry.context.toLowerCase().includes(ph.toLowerCase())
+      ) {
+        issues.push({
+          level: 'warning',
+          field: 'context',
+          message: `Context contains placeholder text: "${ph}"`,
+          explanation: 'Placeholder text may indicate incomplete entry',
+          suggestion: 'Replace with actual context or remove entry',
+        });
+        break;
+      }
+    }
+  }
+
+  // Tool-name validation
+  if (entry.tool_name) {
+    const validTools = ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep', 'Task', 'WebFetch'];
+    if (!validTools.includes(entry.tool_name)) {
+      issues.push({
+        level: 'info',
+        field: 'tool_name',
+        message: `Uncommon tool: "${entry.tool_name}"`,
+        explanation: 'Tool not in standard list - may be valid but unusual',
+        suggestion: 'Verify tool name is correct',
+      });
+    }
+  }
+
+  return issues;
+}
+
+// =============================================================================
+// ANOMALY DETECTION (Level 3)
+// =============================================================================
+
+function detectAnomalies(entry, allEntries) {
+  const entries = Array.isArray(allEntries) ? allEntries : [];
+  const issues = [];
+
+  // Rapid feedback burst (more than 5 in 1 minute)
+  if (entry.timestamp && entries.length > 0) {
+    const entryTime = new Date(entry.timestamp);
+    const recentEntries = entries.filter((e) => {
+      const t = new Date(e.timestamp);
+      return Math.abs(entryTime - t) < 60000; // 1 minute
+    });
+
+    if (recentEntries.length > 5) {
+      issues.push({
+        level: 'warning',
+        type: 'anomaly',
+        message: 'Feedback burst detected',
+        explanation: `${recentEntries.length} entries within 1 minute - unusual pattern`,
+        suggestion: 'Verify this is not automated noise or duplicate entries',
+      });
+    }
+  }
+
+  // Same feedback repeated exactly (duplicate detection)
+  if (entry.context && entries.length > 0) {
+    const duplicates = entries.filter(
+      (e) =>
+        e.context === entry.context &&
+        e.signal === entry.signal &&
+        e.tool_name === entry.tool_name
+    );
+
+    if (duplicates.length > 0) {
+      issues.push({
+        level: 'warning',
+        type: 'anomaly',
+        message: 'Duplicate feedback entry',
+        explanation: `Found ${duplicates.length} identical entries`,
+        suggestion: 'Consider deduplication or review capture logic',
+      });
+    }
+  }
+
+  // Feedback balance check (session imbalance)
+  if (entries.length >= 10) {
+    const positiveCount = entries.filter((e) => e.signal === 'positive').length;
+    const ratio = positiveCount / entries.length;
+
+    if (ratio > 0.95) {
+      issues.push({
+        level: 'info',
+        type: 'anomaly',
+        message: 'Feedback heavily skewed positive',
+        explanation: `${(ratio * 100).toFixed(1)}% positive - may indicate capture bias`,
+        suggestion: 'Review if negative cases are being properly captured',
+      });
+    } else if (ratio < 0.05) {
+      issues.push({
+        level: 'warning',
+        type: 'anomaly',
+        message: 'Feedback heavily skewed negative',
+        explanation: `${((1 - ratio) * 100).toFixed(1)}% negative - unusual pattern`,
+        suggestion: 'Check for systematic issues or misconfigured error detection',
+      });
+    }
+  }
+
+  // Sensitive data leakage detection
+  if (entry.context) {
+    const sensitivePatterns = [
+      /api[_-]?key/i,
+      /password/i,
+      /secret/i,
+      /token/i,
+      /bearer/i,
+      /\b[A-Za-z0-9]{32,}\b/, // Long alphanumeric strings (possible keys)
+    ];
+
+    for (const pattern of sensitivePatterns) {
+      if (pattern.test(entry.context)) {
+        issues.push({
+          level: 'error',
+          type: 'security',
+          message: 'Potential sensitive data in context',
+          explanation: `Pattern matched: ${pattern.toString()}`,
+          suggestion: 'Redact sensitive information before logging',
+        });
+        break;
+      }
+    }
+  }
+
+  return issues;
+}
+
+// =============================================================================
+// SELF-CORRECTION (Level 4)
+// =============================================================================
+
+function generateCorrections(entry, issues) {
+  const corrections = [];
+
+  for (const issue of issues) {
+    if (issue.level === 'error') {
+      // Auto-correct reward to match signal
+      if (issue.field === 'reward' && entry.signal) {
+        const correctedReward =
+          entry.signal === 'positive' ? 1 : entry.signal === 'negative' ? -1 : 0;
+        corrections.push({
+          field: 'reward',
+          original: entry.reward,
+          corrected: correctedReward,
+          reason: 'Auto-corrected to match signal type',
+        });
+      }
+
+      // Auto-add missing timestamp
+      if (issue.field === 'timestamp' && !entry.timestamp) {
+        corrections.push({
+          field: 'timestamp',
+          original: null,
+          corrected: new Date().toISOString(),
+          reason: 'Added missing timestamp',
+        });
+      }
+    }
+  }
+
+  return corrections;
+}
+
+function applyCorrections(entry, corrections) {
+  const corrected = { ...entry };
+  for (const c of corrections) {
+    corrected[c.field] = c.corrected;
+  }
+  corrected._corrected = true;
+  corrected._corrections = corrections;
+  return corrected;
+}
+
+// =============================================================================
+// MAIN VALIDATION PIPELINE
+// =============================================================================
+
+function validateEntry(entry, allEntries) {
+  const entries = Array.isArray(allEntries) ? allEntries : [];
+  const result = {
+    valid: true,
+    entry,
+    issues: [],
+    corrections: [],
+    correctedEntry: null,
+  };
+
+  // Level 1: Schema
+  result.issues.push(...validateSchema(entry));
+
+  // Level 2: Semantics
+  result.issues.push(...validateSemantics(entry));
+
+  // Level 3: Anomalies
+  result.issues.push(...detectAnomalies(entry, entries));
+
+  // Level 4: Self-correction
+  result.corrections = generateCorrections(entry, result.issues);
+
+  // Determine validity (errors make entry invalid)
+  const hasErrors = result.issues.some((i) => i.level === 'error');
+  result.valid = !hasErrors;
+
+  // Apply corrections if available
+  if (result.corrections.length > 0) {
+    result.correctedEntry = applyCorrections(entry, result.corrections);
+  }
+
+  return result;
+}
+
+// =============================================================================
+// FEEDBACK LOG UTILITIES
+// =============================================================================
+
+function loadFeedbackLog() {
+  const { FEEDBACK_LOG } = getFeedbackPaths();
+  if (!fs.existsSync(FEEDBACK_LOG)) return [];
+
+  const content = fs.readFileSync(FEEDBACK_LOG, 'utf8');
+  return content
+    .trim()
+    .split('\n')
+    .filter((line) => line.trim())
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter((e) => e !== null);
+}
+
+// =============================================================================
+// AUDIT MODE
+// =============================================================================
+
+function auditFeedbackLog() {
+  const { VALIDATION_LOG, QUALITY_REPORT } = getFeedbackPaths();
+  console.log('Auditing feedback log...\n');
+
+  const entries = loadFeedbackLog();
+  if (entries.length === 0) {
+    console.log('No entries to audit.');
+    return;
+  }
+
+  const results = {
+    total: entries.length,
+    valid: 0,
+    invalid: 0,
+    corrected: 0,
+    issuesByLevel: { error: 0, warning: 0, info: 0 },
+    issuesByField: {},
+  };
+
+  const validationIssues = [];
+
+  for (const entry of entries) {
+    const validation = validateEntry(entry, entries);
+
+    if (validation.valid) {
+      results.valid++;
+    } else {
+      results.invalid++;
+    }
+
+    if (validation.corrections.length > 0) {
+      results.corrected++;
+    }
+
+    for (const issue of validation.issues) {
+      results.issuesByLevel[issue.level] = (results.issuesByLevel[issue.level] || 0) + 1;
+      if (issue.field) {
+        results.issuesByField[issue.field] = (results.issuesByField[issue.field] || 0) + 1;
+      }
+
+      validationIssues.push({
+        timestamp: entry.timestamp,
+        entryId: entry.id,
+        ...issue,
+      });
+    }
+  }
+
+  // Save validation issues log
+  if (validationIssues.length > 0) {
+    const dir = getFeedbackDir();
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    const issueLog = validationIssues.map((i) => JSON.stringify(i)).join('\n');
+    fs.writeFileSync(VALIDATION_LOG, issueLog + '\n');
+  }
+
+  // Save quality report
+  const report = {
+    ...results,
+    validityRate: ((results.valid / results.total) * 100).toFixed(2) + '%',
+    auditedAt: new Date().toISOString(),
+  };
+  const dir = getFeedbackDir();
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(QUALITY_REPORT, JSON.stringify(report, null, 2) + '\n');
+
+  // Print summary
+  console.log(`Total entries: ${results.total}`);
+  console.log(`Valid: ${results.valid} (${report.validityRate})`);
+  console.log(`Invalid: ${results.invalid}`);
+  console.log(`Auto-correctable: ${results.corrected}`);
+  console.log('\nIssues by level:');
+  console.log(`  Errors: ${results.issuesByLevel.error || 0}`);
+  console.log(`  Warnings: ${results.issuesByLevel.warning || 0}`);
+  console.log(`  Info: ${results.issuesByLevel.info || 0}`);
+
+  if (Object.keys(results.issuesByField).length > 0) {
+    console.log('\nTop issue fields:');
+    const sorted = Object.entries(results.issuesByField)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5);
+    for (const [field, count] of sorted) {
+      console.log(`  ${field}: ${count}`);
+    }
+  }
+
+  console.log(`\nValidation issues saved to: ${VALIDATION_LOG}`);
+  console.log(`Quality report saved to: ${QUALITY_REPORT}`);
+}
+
+function showStats() {
+  const { QUALITY_REPORT } = getFeedbackPaths();
+  if (!fs.existsSync(QUALITY_REPORT)) {
+    console.log('No quality report found. Run --audit first.');
+    return;
+  }
+
+  const report = JSON.parse(fs.readFileSync(QUALITY_REPORT, 'utf8'));
+  console.log('Feedback Quality Statistics\n');
+  console.log(JSON.stringify(report, null, 2));
+}
+
+// =============================================================================
+// CLI ENTRY POINT
+// =============================================================================
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  if (args.includes('--audit')) {
+    auditFeedbackLog();
+  } else if (args.includes('--stats')) {
+    showStats();
+  } else {
+    // Read from stdin (piped input)
+    let input = '';
+
+    if (!process.stdin.isTTY) {
+      for await (const chunk of process.stdin) {
+        input += chunk;
+      }
+    }
+
+    if (input.trim()) {
+      try {
+        const entry = JSON.parse(input);
+        const allEntries = loadFeedbackLog();
+        const result = validateEntry(entry, allEntries);
+
+        if (result.valid) {
+          const output = result.correctedEntry || result.entry;
+          console.log(JSON.stringify(output));
+        } else {
+          console.error('[VALIDATION] Issues found:');
+          for (const issue of result.issues) {
+            console.error(`  [${issue.level}] ${issue.message}`);
+          }
+          console.log(JSON.stringify(result.correctedEntry || result.entry));
+        }
+      } catch (e) {
+        console.error(`[VALIDATION] Invalid JSON: ${e.message}`);
+        process.exit(1);
+      }
+    } else {
+      console.log('Feedback Data Quality Validator');
+      console.log('\nUsage:');
+      console.log("  echo '{\"signal\":\"positive\",...}' | node validate-feedback.js");
+      console.log('  node validate-feedback.js --audit   # Audit existing log');
+      console.log('  node validate-feedback.js --stats   # Show statistics');
+    }
+  }
+}
+
+// =============================================================================
+// MODULE EXPORTS
+// =============================================================================
+
+module.exports = {
+  validateEntry,
+  validateSchema,
+  validateSemantics,
+  detectAnomalies,
+  generateCorrections,
+  applyCorrections,
+  loadFeedbackLog,
+  auditFeedbackLog,
+  showStats,
+};
+
+// Run CLI only when invoked directly
+if (require.main === module) {
+  main().catch(console.error);
+}

package/scripts/vector-store.js

@@ -0,0 +1,100 @@
+'use strict';
+
+const path = require('path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_LANCE_DIR = path.join(PROJECT_ROOT, '.claude', 'memory', 'feedback', 'lancedb');
+
+// Module-level cache — prevents re-importing on every upsertFeedback() call
+// First ESM import takes ~200ms; second is instant from cache.
+let _lancedb = null;
+let _pipeline = null;
+const TABLE_NAME = 'rlhf_memories';
+
+async function getLanceDB() {
+  if (!_lancedb) {
+    _lancedb = await import('@lancedb/lancedb');
+  }
+  return _lancedb;
+}
+
+async function getEmbeddingPipeline() {
+  if (!_pipeline) {
+    const { pipeline } = await import('@huggingface/transformers');
+    _pipeline = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+      quantized: true,
+    });
+  }
+  return _pipeline;
+}
+
+// Stub embed support for unit tests — avoids HuggingFace ONNX model download.
+// Set RLHF_VECTOR_STUB_EMBED=true to get a deterministic 384-dim unit vector.
+// The real embed() is used in production and integration tests
+// (gated by absence of this env var).
+async function embed(text) {
+  if (process.env.RLHF_VECTOR_STUB_EMBED === 'true') {
+    // Deterministic 384-dim unit vector: first element = 1.0, rest = 0.0
+    const stub = Array(384).fill(0);
+    stub[0] = 1.0;
+    return stub;
+  }
+  const pipe = await getEmbeddingPipeline();
+  const output = await pipe(text, { pooling: 'mean', normalize: true });
+  return Array.from(output.data); // Float32Array -> plain number[] for LanceDB Arrow serialization
+}
+
+async function upsertFeedback(feedbackEvent) {
+  const lanceDir = process.env.RLHF_FEEDBACK_DIR
+    ? path.join(process.env.RLHF_FEEDBACK_DIR, 'lancedb')
+    : DEFAULT_LANCE_DIR;
+
+  const { connect } = await getLanceDB();
+  const db = await connect(lanceDir);
+
+  const textForEmbedding = [
+    feedbackEvent.context || '',
+    (feedbackEvent.tags || []).join(' '),
+    feedbackEvent.whatWentWrong || '',
+    feedbackEvent.whatWorked || '',
+  ].filter(Boolean).join('. ');
+
+  const vector = await embed(textForEmbedding);
+
+  const record = {
+    id: feedbackEvent.id,
+    text: textForEmbedding,
+    vector,
+    signal: feedbackEvent.signal,
+    tags: (feedbackEvent.tags || []).join(','),
+    timestamp: feedbackEvent.timestamp,
+    context: feedbackEvent.context || '',
+  };
+
+  const tableNames = await db.tableNames();
+  if (tableNames.includes(TABLE_NAME)) {
+    const table = await db.openTable(TABLE_NAME);
+    await table.add([record]);
+  } else {
+    await db.createTable(TABLE_NAME, [record]);
+  }
+}
+
+async function searchSimilar(queryText, limit = 5) {
+  const lanceDir = process.env.RLHF_FEEDBACK_DIR
+    ? path.join(process.env.RLHF_FEEDBACK_DIR, 'lancedb')
+    : DEFAULT_LANCE_DIR;
+
+  const { connect } = await getLanceDB();
+  const db = await connect(lanceDir);
+
+  const tableNames = await db.tableNames();
+  if (!tableNames.includes(TABLE_NAME)) return [];
+
+  const vector = await embed(queryText);
+  const table = await db.openTable(TABLE_NAME);
+  const results = await table.search(vector).limit(limit).toArray();
+  return results;
+}
+
+module.exports = { upsertFeedback, searchSimilar, TABLE_NAME };