rlhf-feedback-loop 0.6.11 → 0.6.12

package/scripts/feedback-loop.js

@@ -14,6 +14,9 @@ const {
   prepareForStorage,
   parseTimestamp,
 } = require('./feedback-schema');
+const {
+  buildClarificationMessage,
+} = require('./feedback-quality');
 const {
   buildRubricEvaluation,
 } = require('./rubric-engine');
@@ -30,6 +33,7 @@ const DOMAIN_CATEGORIES = [
 ];
 
 const HOME = process.env.HOME || process.env.USERPROFILE || '';
+const pendingBackgroundSideEffects = new Set();
 
 function getFeedbackPaths() {
   if (process.env.RLHF_FEEDBACK_DIR) {
@@ -85,6 +89,14 @@ function getVectorStoreModule() {
   }
 }
 
+function getRiskScorerModule() {
+  try {
+    return require('./risk-scorer');
+  } catch {
+    return null;
+  }
+}
+
 function getSelfAuditModule() {
   try {
     return require('./rlaif-self-audit');
@@ -104,6 +116,34 @@ function appendJSONL(filePath, record) {
   fs.appendFileSync(filePath, `${JSON.stringify(record)}\n`);
 }
 
+function trackBackgroundSideEffect(taskPromise) {
+  if (!taskPromise || typeof taskPromise.then !== 'function') {
+    return null;
+  }
+
+  let tracked;
+  tracked = Promise.resolve(taskPromise)
+    .catch(() => {
+      // Non-critical side effects should never fail the primary feedback write.
+    })
+    .finally(() => {
+      pendingBackgroundSideEffects.delete(tracked);
+    });
+
+  pendingBackgroundSideEffects.add(tracked);
+  return tracked;
+}
+
+async function waitForBackgroundSideEffects() {
+  while (pendingBackgroundSideEffects.size > 0) {
+    await Promise.allSettled([...pendingBackgroundSideEffects]);
+  }
+}
+
+function getPendingBackgroundSideEffectCount() {
+  return pendingBackgroundSideEffects.size;
+}
+
 function readJSONL(filePath) {
   if (!fs.existsSync(filePath)) return [];
   const raw = fs.readFileSync(filePath, 'utf-8').trim();
@@ -290,15 +330,41 @@ function buildSequenceFeatures(recentEntries, currentEntry) {
   };
 }
 
-function appendSequence(feedbackEvent, paths) {
+function appendSequence(historyEntries, feedbackEvent, paths, outcome = {}) {
   const sequencePath = path.join(paths.FEEDBACK_DIR, 'feedback-sequences.jsonl');
-  const recent = readJSONL(paths.FEEDBACK_LOG_PATH).slice(-SEQUENCE_WINDOW);
+  const recent = Array.isArray(historyEntries) ? historyEntries.slice(-SEQUENCE_WINDOW) : [];
   const features = buildSequenceFeatures(recent, feedbackEvent);
+  const rubric = feedbackEvent.rubric || null;
+  const filePaths = feedbackEvent.richContext && Array.isArray(feedbackEvent.richContext.filePaths)
+    ? feedbackEvent.richContext.filePaths
+    : [];
+  const accepted = outcome.accepted === true;
+  const targetRisk = feedbackEvent.signal === 'negative' || !accepted ? 1 : 0;
   const entry = {
     id: `seq_${Date.now()}`,
     timestamp: new Date().toISOString(),
     targetReward: feedbackEvent.signal === 'positive' ? 1 : -1,
     targetTags: feedbackEvent.tags,
+    accepted,
+    actionType: feedbackEvent.actionType || null,
+    actionReason: feedbackEvent.actionReason || null,
+    context: feedbackEvent.context || '',
+    skill: feedbackEvent.skill || null,
+    domain: feedbackEvent.richContext ? feedbackEvent.richContext.domain : 'general',
+    outcomeCategory: feedbackEvent.richContext ? feedbackEvent.richContext.outcomeCategory : 'unknown',
+    filePathCount: filePaths.length,
+    errorType: feedbackEvent.richContext ? feedbackEvent.richContext.errorType : null,
+    rubric: rubric
+      ? {
+        rubricId: rubric.rubricId || null,
+        weightedScore: rubric.weightedScore,
+        failingCriteria: rubric.failingCriteria || [],
+        failingGuardrails: rubric.failingGuardrails || [],
+        judgeDisagreements: rubric.judgeDisagreements || [],
+      }
+      : null,
+    targetRisk,
+    riskLabel: targetRisk === 1 ? 'high-risk' : 'low-risk',
     features,
     label: feedbackEvent.signal === 'positive' ? 'positive' : 'negative',
   };
@@ -343,7 +409,7 @@ function updateDiversityTracking(feedbackEvent, paths) {
 }
 
 function captureFeedback(params) {
-  const { FEEDBACK_LOG_PATH, MEMORY_LOG_PATH } = getFeedbackPaths();
+  const { FEEDBACK_LOG_PATH, MEMORY_LOG_PATH, FEEDBACK_DIR } = getFeedbackPaths();
   const signal = normalizeSignal(params.signal);
   if (!signal) {
     return {
@@ -411,20 +477,44 @@ function captureFeedback(params) {
 
   // Rich context enrichment (QUAL-02, QUAL-03) — non-blocking
   const feedbackEvent = enrichFeedbackContext(rawFeedbackEvent, params);
+  const historyEntries = readJSONL(FEEDBACK_LOG_PATH).slice(-SEQUENCE_WINDOW);
 
   const summary = loadSummary();
   summary.total += 1;
   summary[signal] += 1;
 
   if (action.type === 'no-action') {
+    const clarification = buildClarificationMessage({
+      signal,
+      context: params.context || '',
+      whatWentWrong: params.whatWentWrong,
+      whatToChange: params.whatToChange,
+      whatWorked: params.whatWorked,
+    });
     summary.rejected += 1;
     summary.lastUpdated = now;
     saveSummary(summary);
     appendJSONL(FEEDBACK_LOG_PATH, feedbackEvent);
+    try {
+      appendSequence(historyEntries, feedbackEvent, getFeedbackPaths(), { accepted: false });
+    } catch {
+      // Sequence tracking failure is non-critical
+    }
+    try {
+      const riskScorer = getRiskScorerModule();
+      if (riskScorer) {
+        riskScorer.trainAndPersistRiskModel(FEEDBACK_DIR);
+      }
+    } catch {
+      // Risk model refresh is non-critical
+    }
     return {
       accepted: false,
+      status: clarification ? 'clarification_required' : 'rejected',
       reason: action.reason,
+      message: clarification ? clarification.message : 'Signal logged, but reusable memory was not created.',
       feedbackEvent,
+      ...(clarification || {}),
     };
   }
 
@@ -437,9 +527,24 @@ function captureFeedback(params) {
       ...feedbackEvent,
       validationIssues: prepared.issues,
     });
+    try {
+      appendSequence(historyEntries, feedbackEvent, getFeedbackPaths(), { accepted: false });
+    } catch {
+      // Sequence tracking failure is non-critical
+    }
+    try {
+      const riskScorer = getRiskScorerModule();
+      if (riskScorer) {
+        riskScorer.trainAndPersistRiskModel(FEEDBACK_DIR);
+      }
+    } catch {
+      // Risk model refresh is non-critical
+    }
     return {
       accepted: false,
+      status: 'rejected',
       reason: `Schema validation failed: ${prepared.issues.join('; ')}`,
+      message: 'Signal logged, but reusable memory was not created.',
       feedbackEvent,
       issues: prepared.issues,
     };
@@ -467,7 +572,7 @@ function captureFeedback(params) {
   // ML side-effects: sequence tracking and diversity (non-blocking — primary write already succeeded)
   const mlPaths = getFeedbackPaths();
   try {
-    appendSequence(feedbackEvent, mlPaths);
+    appendSequence(historyEntries, feedbackEvent, mlPaths, { accepted: true });
   } catch (err) {
     // Sequence tracking failure is non-critical
   }
@@ -479,10 +584,8 @@ function captureFeedback(params) {
 
   // Vector storage side-effect (non-blocking — primary write already succeeded)
   const vectorStore = getVectorStoreModule();
-  if (vectorStore) {
-    vectorStore.upsertFeedback(feedbackEvent).catch(() => {
-      // Non-critical; primary feedback log is the source of truth
-    });
+  if (vectorStore && typeof vectorStore.upsertFeedback === 'function') {
+    trackBackgroundSideEffect(vectorStore.upsertFeedback(feedbackEvent));
   }
 
   // RLAIF self-audit side-effect (non-blocking — 4th enrichment layer)
@@ -491,6 +594,14 @@ function captureFeedback(params) {
     if (sam) sam.selfAuditAndLog(feedbackEvent, mlPaths);
   } catch (_err) { /* non-critical */ }
 
+  // Boosted risk model refresh — local, file-based, and non-blocking
+  try {
+    const riskScorer = getRiskScorerModule();
+    if (riskScorer) {
+      riskScorer.trainAndPersistRiskModel(FEEDBACK_DIR);
+    }
+  } catch (_err) { /* non-critical */ }
+
   // Attribution side-effects — fire-and-forget, never throw
   try {
     const toolName = feedbackEvent.toolName || feedbackEvent.tool_name || 'unknown';
@@ -511,6 +622,8 @@ function captureFeedback(params) {
 
   return {
     accepted: true,
+    status: 'promoted',
+    message: 'Feedback promoted to reusable memory.',
     feedbackEvent,
     memoryRecord,
   };
@@ -519,6 +632,7 @@ function captureFeedback(params) {
 function analyzeFeedback(logPath) {
   const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
   const entries = readJSONL(logPath || FEEDBACK_LOG_PATH);
+  const paths = getFeedbackPaths();
   const skills = {};
   const tags = {};
   const rubricCriteria = {};
@@ -586,6 +700,24 @@ function analyzeFeedback(logPath) {
     recommendations.push('DECLINING trend in last 20 signals; tighten verification before response.');
   }
 
+  let boostedRisk = null;
+  try {
+    const riskScorer = getRiskScorerModule();
+    if (riskScorer) {
+      boostedRisk = riskScorer.getRiskSummary(paths.FEEDBACK_DIR);
+      if (boostedRisk) {
+        boostedRisk.highRiskDomains.slice(0, 2).forEach((bucket) => {
+          recommendations.push(`CHECK high-risk domain '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
+        });
+        boostedRisk.highRiskTags.slice(0, 2).forEach((bucket) => {
+          recommendations.push(`CHECK high-risk tag '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
+        });
+      }
+    }
+  } catch {
+    boostedRisk = null;
+  }
+
   return {
     total,
     totalPositive,
@@ -599,6 +731,7 @@ function analyzeFeedback(logPath) {
       blockedPromotions,
       failingCriteria: rubricCriteria,
     },
+    boostedRisk,
     recommendations,
   };
 }
@@ -699,6 +832,15 @@ function feedbackSummary(recentN = 20) {
     `- Overall approval: ${Math.round(analysis.approvalRate * 100)}%`,
   ];
 
+  if (analysis.boostedRisk) {
+    lines.push(`- Boosted risk base rate: ${Math.round((analysis.boostedRisk.baseRate || 0) * 100)}%`);
+    lines.push(`- Boosted risk mode: ${analysis.boostedRisk.mode}`);
+    if (analysis.boostedRisk.highRiskDomains.length > 0) {
+      const topDomain = analysis.boostedRisk.highRiskDomains[0];
+      lines.push(`- Highest-risk domain: ${topDomain.key} (${Math.round(topDomain.riskRate * 100)}%)`);
+    }
+  }
+
   if (analysis.recommendations.length > 0) {
     lines.push('- Recommendations:');
     analysis.recommendations.slice(0, 5).forEach((r) => lines.push(`  - ${r}`));
@@ -822,6 +964,7 @@ function runTests() {
 
   const bad = captureFeedback({ signal: 'down' });
   assert(!bad.accepted, 'captureFeedback rejects vague negative feedback');
+  assert(bad.needsClarification === true, 'captureFeedback requests clarification for vague negative feedback');
 
   const summary = feedbackSummary(5);
   assert(summary.includes('Feedback Summary'), 'feedbackSummary returns text output');
@@ -848,6 +991,8 @@ module.exports = {
   inferDomain,
   inferOutcome,
   enrichFeedbackContext,
+  waitForBackgroundSideEffects,
+  getPendingBackgroundSideEffectCount,
   get FEEDBACK_LOG_PATH() {
     return getFeedbackPaths().FEEDBACK_LOG_PATH;
   },

package/scripts/feedback-quality.js

@@ -0,0 +1,139 @@
+'use strict';
+
+const GENERIC_PHRASE_RULES = {
+  positive: [
+    /^up$/,
+    /^thumbs?\s*up$/,
+    /^thumbs\s+up$/,
+    /^that worked$/,
+    /^it worked$/,
+    /^worked$/,
+    /^looks good$/,
+    /^looked good$/,
+    /^good job$/,
+    /^good work$/,
+    /^nice work$/,
+    /^perfect$/,
+    /^approved$/,
+    /^lgtm$/,
+  ],
+  negative: [
+    /^down$/,
+    /^thumbs?\s*down$/,
+    /^thumbs\s+down$/,
+    /^that failed$/,
+    /^it failed$/,
+    /^failed$/,
+    /^that was wrong$/,
+    /^wrong$/,
+    /^bad$/,
+    /^fix this$/,
+    /^broken$/,
+  ],
+};
+
+const CLARIFICATION_CONFIG = {
+  positive: {
+    prompt: 'What specifically worked that should be repeated?',
+    example: 'Example: "The agent showed test output before claiming done."',
+    missingFields: ['whatWorked'],
+  },
+  negative: {
+    prompt: 'What failed and what should change next time?',
+    example: 'Example: "It skipped tests and should run npm test before closing the task."',
+    missingFields: ['whatWentWrong', 'whatToChange'],
+  },
+};
+
+function normalizeFeedbackSignal(signal) {
+  const normalized = normalizeFeedbackText(signal);
+  if (['negative', 'down', 'thumbs down', 'thumbsdown', 'bad'].includes(normalized)) {
+    return 'negative';
+  }
+  return 'positive';
+}
+
+function normalizeFeedbackText(value) {
+  return String(value || '')
+    .toLowerCase()
+    .replace(/[_-]+/g, ' ')
+    .replace(/[^\w\s]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function isGenericFeedbackText(value, signal) {
+  const normalized = normalizeFeedbackText(value);
+  if (!normalized) return false;
+  const rules = GENERIC_PHRASE_RULES[signal] || [];
+  return rules.some((pattern) => pattern.test(normalized));
+}
+
+function assessFeedbackActionability(params = {}) {
+  const signal = normalizeFeedbackSignal(params.signal);
+  const primaryFields = signal === 'positive'
+    ? [
+      { name: 'whatWorked', value: params.whatWorked },
+      { name: 'context', value: params.context },
+    ]
+    : [
+      { name: 'whatWentWrong', value: params.whatWentWrong },
+      { name: 'context', value: params.context },
+    ];
+
+  const populated = primaryFields.filter((field) => normalizeFeedbackText(field.value));
+  const specific = populated.find((field) => !isGenericFeedbackText(field.value, signal));
+
+  if (specific) {
+    return {
+      promotable: true,
+      signal,
+      sourceField: specific.name,
+      prompt: null,
+      example: null,
+      missingFields: [],
+      issue: null,
+      isGenericContext: false,
+    };
+  }
+
+  const config = CLARIFICATION_CONFIG[signal];
+  const issue = populated.length > 0 ? 'generic' : 'missing';
+
+  return {
+    promotable: false,
+    signal,
+    sourceField: null,
+    prompt: config.prompt,
+    example: config.example,
+    missingFields: config.missingFields,
+    issue,
+    isGenericContext: populated.some((field) => field.name === 'context'),
+  };
+}
+
+function buildClarificationMessage(params = {}) {
+  const assessment = assessFeedbackActionability(params);
+  if (assessment.promotable) return null;
+
+  const intro = assessment.signal === 'positive'
+    ? 'Positive signal logged, but it is not specific enough to promote to reusable memory.'
+    : 'Negative signal logged, but it is not specific enough to promote to reusable memory.';
+
+  return {
+    needsClarification: true,
+    prompt: assessment.prompt,
+    example: assessment.example,
+    missingFields: assessment.missingFields,
+    message: `${intro} ${assessment.prompt}`,
+  };
+}
+
+module.exports = {
+  GENERIC_PHRASE_RULES,
+  normalizeFeedbackSignal,
+  normalizeFeedbackText,
+  isGenericFeedbackText,
+  assessFeedbackActionability,
+  buildClarificationMessage,
+};

package/scripts/feedback-schema.js

@@ -12,6 +12,9 @@ const GENERIC_TAGS = new Set(['feedback', 'positive', 'negative']);
 const MIN_CONTENT_LENGTH = 20;
 const VALID_TITLE_PREFIXES = ['SUCCESS:', 'MISTAKE:', 'LEARNING:', 'PREFERENCE:'];
 const VALID_CATEGORIES = new Set(['error', 'learning', 'preference']);
+const {
+  assessFeedbackActionability,
+} = require('./feedback-quality');
 
 function validateFeedbackMemory(memory) {
   const issues = [];
@@ -112,8 +115,16 @@ function resolveFeedbackAction(params) {
     : [];
 
   if (signal === 'negative') {
-    if (!whatWentWrong && !context) {
-      return { type: 'no-action', reason: 'Negative feedback without context — cannot determine what went wrong' };
+    const actionability = assessFeedbackActionability({
+      signal: 'negative',
+      context,
+      whatWentWrong,
+    });
+    if (!actionability.promotable) {
+      const reason = actionability.issue === 'missing'
+        ? 'Negative feedback without context — cannot determine what went wrong'
+        : 'Negative feedback is too vague to promote — describe what failed in one sentence';
+      return { type: 'no-action', reason };
     }
 
     const content = [
@@ -157,8 +168,16 @@ function resolveFeedbackAction(params) {
       return { type: 'no-action', reason: `Rubric gate prevented promotion: ${reasons}` };
     }
 
-    if (!whatWorked && !context) {
-      return { type: 'no-action', reason: 'Positive feedback without context — cannot determine what worked' };
+    const actionability = assessFeedbackActionability({
+      signal: 'positive',
+      context,
+      whatWorked,
+    });
+    if (!actionability.promotable) {
+      const reason = actionability.issue === 'missing'
+        ? 'Positive feedback without context — cannot determine what worked'
+        : 'Positive feedback is too vague to promote — describe what worked in one sentence';
+      return { type: 'no-action', reason };
     }
 
     const content = whatWorked ? `What worked: ${whatWorked}` : `Approach: ${context}`;
@@ -246,6 +265,13 @@ function runTests() {
   const bareThumbsDown = resolveFeedbackAction({ signal: 'negative' });
   assert(bareThumbsDown.type === 'no-action', 'bare negative feedback becomes no-action');
 
+  const vagueThumbsUp = resolveFeedbackAction({
+    signal: 'positive',
+    context: 'thumbs up',
+    tags: ['verification'],
+  });
+  assert(vagueThumbsUp.type === 'no-action', 'generic positive context becomes no-action');
+
   const fullNegative = resolveFeedbackAction({
     signal: 'negative',
     context: 'Pushed code with no tests',
@@ -267,7 +293,7 @@ function runTests() {
 
   const blockedPositive = resolveFeedbackAction({
     signal: 'positive',
-    whatWorked: 'Looked correct',
+    whatWorked: 'Manual approval happened without evidence',
     tags: ['testing'],
     rubricEvaluation: {
       promotionEligible: false,

package/scripts/feedback-to-memory.js

@@ -25,6 +25,7 @@
 'use strict';
 
 const { resolveFeedbackAction, prepareForStorage } = require('./feedback-schema');
+const { buildClarificationMessage } = require('./feedback-quality');
 
 function convertFeedbackToMemory(params) {
   const action = resolveFeedbackAction({
@@ -37,7 +38,18 @@ function convertFeedbackToMemory(params) {
   });
 
   if (!action || action.type === 'no-action') {
-    return { ok: false, reason: action ? action.reason : 'Unknown action resolution failure' };
+    const clarification = buildClarificationMessage({
+      signal: params.signal,
+      context: params.context || '',
+      whatWentWrong: params.whatWentWrong,
+      whatToChange: params.whatToChange,
+      whatWorked: params.whatWorked,
+    });
+    return {
+      ok: false,
+      reason: action ? action.reason : 'Unknown action resolution failure',
+      ...(clarification || {}),
+    };
   }
 
   const prep = prepareForStorage(action.memory);

package/scripts/hook-auto-capture.sh

@@ -17,6 +17,12 @@ capture_and_report() {
 
   # Capture feedback (verbose output already shows IDs, signal, storage)
   node "$CAPTURE" --feedback="$SIGNAL" --context="$PROMPT" --tags="auto-capture,hook"
+  local CAPTURE_STATUS=$?
+
+  if [ "$CAPTURE_STATUS" -eq 2 ]; then
+    echo "Reusable memory status: signal logged only. Add one specific sentence so the MCP can promote it."
+    echo ""
+  fi
 
   # Show storage proof
   echo ""

package/scripts/hook-stop-self-score.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Claude Code / Amp Stop hook — autonomous self-scoring after every agent turn
+# Fires after the agent completes a response. Runs selfAuditAndLog to produce
+# a RLAIF self-score entry in self-score-log.jsonl.
+#
+# Environment variables available in Stop hooks:
+#   CLAUDE_STOP_REASON   — why the agent stopped (e.g., "end_turn", "tool_use")
+#   CLAUDE_TOOL_OUTPUT   — last tool output (if any)
+#
+# This hook is NON-BLOCKING — it exits 0 regardless of errors.
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+RLHF_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Run the self-score via Node.js — sync, no API calls, ~5ms
+node -e '
+  "use strict";
+  const path = require("path");
+
+  // Resolve modules relative to RLHF package root
+  const rlhfRoot = process.env.RLHF_ROOT;
+  const { selfAuditAndLog } = require(path.join(rlhfRoot, "scripts", "rlaif-self-audit"));
+  const { getFeedbackPaths } = require(path.join(rlhfRoot, "scripts", "feedback-loop"));
+
+  const stopReason = process.env.CLAUDE_STOP_REASON || "unknown";
+
+  // Build a minimal feedback event for self-scoring
+  const feedbackEvent = {
+    id: `stop_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`,
+    signal: "positive",
+    context: `Agent turn completed (stop_reason: ${stopReason}). Autonomous self-score checkpoint.`,
+    tags: ["stop-hook", "auto-score"],
+    whatWorked: null,
+    whatWentWrong: null,
+    whatToChange: null,
+    rubric: null,
+  };
+
+  const paths = getFeedbackPaths();
+  const result = selfAuditAndLog(feedbackEvent, paths);
+
+  // Output minimal JSON for hook response (non-blocking)
+  process.stdout.write(JSON.stringify({
+    hookSpecificOutput: {
+      hookEventName: "Stop",
+      additionalContext: `Self-score: ${result.score} (${result.constraints.filter(c => c.passed).length}/${result.constraints.length} constraints passed)`
+    }
+  }));
+' 2>/dev/null || true
+
+exit 0