thumbgate 1.16.3 → 1.16.4

package/scripts/background-agent-governance.js

@@ -0,0 +1,229 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Background Agent Governance — the missing layer for Ramp/Ona-style agent stacks.
+ *
+ * Background agents run unattended (writing 57% of PRs at Ramp). They need:
+ * 1. Run tracking — what did each agent run do?
+ * 2. Governance gate — should this PR/action be allowed based on past failures?
+ * 3. Post-run audit — auto-capture feedback from CI results
+ * 4. Governance report — "X runs, Y blocked, Z lessons learned"
+ *
+ * Integrates with: MCP server, gates engine, org dashboard, lesson inference.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { resolveFeedbackDir } = require('./feedback-paths');
+const { ensureParentDir, readJsonl } = require('./fs-utils');
+
+const RUNS_FILE = 'agent-runs.jsonl';
+
+function getFeedbackDir(feedbackDir) { return resolveFeedbackDir({ feedbackDir }); }
+function getRunsPath(feedbackDir) { return path.join(getFeedbackDir(feedbackDir), RUNS_FILE); }
+
+// ---------------------------------------------------------------------------
+// 1. Run Tracking
+// ---------------------------------------------------------------------------
+
+/**
+ * Record a background agent run.
+ * Called when a background agent starts or completes a task.
+ */
+function recordAgentRun({ agentId, runType, source, branch, prNumber, status, gatesChecked, gatesBlocked, filesChanged, ciPassed, duration, metadata } = {}, feedbackDir) {
+  const runsPath = getRunsPath(feedbackDir);
+  ensureParentDir(runsPath);
+  const run = {
+    id: `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+    timestamp: new Date().toISOString(),
+    agentId: agentId || 'unknown',
+    runType: runType || 'unknown', // 'pr', 'fix', 'refactor', 'ci-repair', 'migration'
+    source: source || 'background', // 'background', 'triggered', 'scheduled', 'manual'
+    branch: branch || null,
+    prNumber: prNumber || null,
+    status: status || 'started', // 'started', 'completed', 'blocked', 'failed'
+    gatesChecked: gatesChecked || 0,
+    gatesBlocked: gatesBlocked || 0,
+    filesChanged: filesChanged || 0,
+    ciPassed: ciPassed === undefined ? null : ciPassed,
+    durationMs: duration || null,
+    metadata: metadata || {},
+  };
+  fs.appendFileSync(runsPath, JSON.stringify(run) + '\n');
+  return run;
+}
+
+// ---------------------------------------------------------------------------
+// 2. Governance Gate — pre-run check
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a background agent run should proceed based on governance rules.
+ * Returns { allowed, blockers, warnings, governanceScore }.
+ */
+function checkRunGovernance({ agentId, runType, branch, filesChanged } = {}, feedbackDir) {
+  const runs = readJsonl(getRunsPath(feedbackDir));
+  const blockers = [];
+  const warnings = [];
+
+  // Rule 1: Block if this agent has > 50% failure rate in last 10 runs
+  const agentRuns = runs.filter((r) => r.agentId === agentId).slice(-10);
+  const failedRuns = agentRuns.filter((r) => r.status === 'failed' || r.status === 'blocked');
+  if (agentRuns.length >= 5 && failedRuns.length / agentRuns.length > 0.5) {
+    blockers.push({ rule: 'high_failure_rate', message: `Agent ${agentId} has ${failedRuns.length}/${agentRuns.length} failed runs (>50%)`, severity: 'critical' });
+  }
+
+  // Rule 2: Warn if agent has been blocked by gates in recent runs
+  const recentBlocked = agentRuns.filter((r) => r.gatesBlocked > 0);
+  if (recentBlocked.length >= 3) {
+    warnings.push({ rule: 'repeated_gate_blocks', message: `Agent ${agentId} has been gate-blocked in ${recentBlocked.length} recent runs`, severity: 'warning' });
+  }
+
+  // Rule 3: Block if targeting protected branch without CI passing
+  if (branch && /^(main|master|develop)$/.test(branch)) {
+    warnings.push({ rule: 'protected_branch', message: `Run targets protected branch "${branch}" — CI must pass before merge`, severity: 'warning' });
+  }
+
+  // Rule 4: Warn if too many files changed (large blast radius)
+  if (filesChanged > 20) {
+    warnings.push({ rule: 'large_blast_radius', message: `${filesChanged} files changed — consider splitting into smaller PRs`, severity: 'warning' });
+  }
+
+  const governanceScore = Math.max(0, 100 - blockers.length * 40 - warnings.length * 10);
+
+  return {
+    allowed: blockers.length === 0,
+    blockers,
+    warnings,
+    governanceScore,
+    checkedAt: new Date().toISOString(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// 3. Post-Run Audit — auto-capture feedback from CI
+// ---------------------------------------------------------------------------
+
+/**
+ * Auto-capture feedback from a completed background agent run.
+ * Converts CI pass/fail into structured feedback for the learning loop.
+ */
+function auditCompletedRun({ runId, agentId, ciPassed, ciOutput, prNumber, branch, filesChanged } = {}, feedbackDir) {
+  const signal = ciPassed ? 'positive' : 'negative';
+  const context = ciPassed
+    ? `Background agent run ${runId || 'unknown'} completed successfully. PR #${prNumber || '?'} on ${branch || '?'}. ${filesChanged || 0} files changed. CI passed.`
+    : `Background agent run ${runId || 'unknown'} failed. PR #${prNumber || '?'} on ${branch || '?'}. ${filesChanged || 0} files changed. CI failed.`;
+
+  const whatWentWrong = !ciPassed && ciOutput ? ciOutput.slice(0, 500) : null;
+
+  // Record the completed run
+  const run = recordAgentRun({
+    agentId,
+    runType: 'pr',
+    source: 'background',
+    branch,
+    prNumber,
+    status: ciPassed ? 'completed' : 'failed',
+    filesChanged,
+    ciPassed,
+  }, feedbackDir);
+
+  // Auto-capture feedback
+  let feedbackResult = null;
+  try {
+    const { captureFeedback } = require('./feedback-loop');
+    feedbackResult = captureFeedback({
+      signal: ciPassed ? 'up' : 'down',
+      context,
+      whatWentWrong,
+      whatWorked: ciPassed ? `Agent successfully completed PR #${prNumber || '?'}` : undefined,
+      tags: ['background-agent', ciPassed ? 'ci-pass' : 'ci-fail', `agent:${agentId || 'unknown'}`],
+    });
+  } catch { /* feedback capture is non-critical */ }
+
+  return { run, feedbackResult, signal, context };
+}
+
+// ---------------------------------------------------------------------------
+// 4. Governance Report
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate a governance report for background agent runs.
+ * Shows: total runs, blocked, pass rate, top failing agents, lessons learned.
+ */
+function generateGovernanceReport({ periodHours = 24, feedbackDir } = {}) {
+  const runs = readJsonl(getRunsPath(feedbackDir));
+  const cutoff = Date.now() - periodHours * 60 * 60 * 1000;
+  const recent = runs.filter((r) => new Date(r.timestamp).getTime() > cutoff);
+
+  const total = recent.length;
+  const completed = recent.filter((r) => r.status === 'completed').length;
+  const failed = recent.filter((r) => r.status === 'failed').length;
+  const blocked = recent.filter((r) => r.status === 'blocked').length;
+  const started = recent.filter((r) => r.status === 'started').length;
+
+  const passRate = (completed + failed) > 0 ? Math.round((completed / (completed + failed)) * 1000) / 10 : 0;
+  const totalGatesChecked = recent.reduce((s, r) => s + (r.gatesChecked || 0), 0);
+  const totalGatesBlocked = recent.reduce((s, r) => s + (r.gatesBlocked || 0), 0);
+
+  // Per-agent breakdown
+  const byAgent = {};
+  for (const r of recent) {
+    if (!byAgent[r.agentId]) byAgent[r.agentId] = { completed: 0, failed: 0, blocked: 0, total: 0 };
+    byAgent[r.agentId].total++;
+    if (r.status === 'completed') byAgent[r.agentId].completed++;
+    if (r.status === 'failed') byAgent[r.agentId].failed++;
+    if (r.status === 'blocked') byAgent[r.agentId].blocked++;
+  }
+
+  const agentSummaries = Object.entries(byAgent).map(([id, counts]) => ({
+    agentId: id,
+    ...counts,
+    passRate: (counts.completed + counts.failed) > 0 ? Math.round((counts.completed / (counts.completed + counts.failed)) * 1000) / 10 : 0,
+  })).sort((a, b) => a.passRate - b.passRate);
+
+  // By run type
+  const byType = {};
+  for (const r of recent) {
+    if (!byType[r.runType]) byType[r.runType] = 0;
+    byType[r.runType]++;
+  }
+
+  return {
+    periodHours,
+    total, completed, failed, blocked, started,
+    passRate,
+    gatesChecked: totalGatesChecked,
+    gatesBlocked: totalGatesBlocked,
+    agents: agentSummaries,
+    topFailingAgent: agentSummaries.length > 0 && agentSummaries[0].passRate < 80 ? agentSummaries[0] : null,
+    byType,
+    generatedAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * Format governance report as a human-readable string.
+ */
+function formatGovernanceReport(report) {
+  const lines = [
+    `Background Agent Governance Report (${report.periodHours}h)`,
+    `Total runs: ${report.total} | Completed: ${report.completed} | Failed: ${report.failed} | Blocked: ${report.blocked}`,
+    `Pass rate: ${report.passRate}%`,
+    `Gates checked: ${report.gatesChecked} | Gates blocked: ${report.gatesBlocked}`,
+  ];
+  if (report.topFailingAgent) {
+    lines.push(`Top failing agent: ${report.topFailingAgent.agentId} (${report.topFailingAgent.passRate}% pass rate)`);
+  }
+  if (Object.keys(report.byType).length > 0) {
+    lines.push(`Run types: ${Object.entries(report.byType).map(([t, c]) => `${t}:${c}`).join(', ')}`);
+  }
+  return lines.join('\n');
+}
+
+module.exports = {
+  recordAgentRun, checkRunGovernance, auditCompletedRun,
+  generateGovernanceReport, formatGovernanceReport, getRunsPath,
+};

package/scripts/dashboard.js

@@ -39,9 +39,15 @@ const { buildPredictiveInsights } = loadOptionalModule('./predictive-insights',
 const { routeProfile } = require('./profile-router');
 const { getSettingsStatus } = require('./settings-hierarchy');
 const { summarizeWorkflowRuns } = require('./workflow-runs');
+const { generateGovernanceReport } = require('./background-agent-governance');
 const { searchLessons } = require('./lesson-search');
 const { getInterventionPolicySummary } = require('./intervention-policy');
-const { computeDecisionMetrics } = require('./decision-journal');
+const {
+  DECISION_LOG_FILENAME,
+  computeDecisionMetrics,
+  readDecisionLog,
+} = require('./decision-journal');
+const { analyzeFeedback } = require('./feedback-loop');
 
 const PROJECT_ROOT = path.join(__dirname, '..');
 const DEFAULT_GATES_PATH = path.join(PROJECT_ROOT, 'config', 'gates', 'default.json');
@@ -1308,6 +1314,183 @@ function computeHarnessOverview(feedbackDir, entries) {
   };
 }
 
+function remediationPriority(type) {
+  const priorities = {
+    'trend-declining': 90,
+    'trend-degrading': 85,
+    'high-risk-domain': 80,
+    'high-risk-tag': 78,
+    'skill-improve': 72,
+    'delegation-reduce': 68,
+    'delegation-policy-review': 66,
+    'diagnose-failure-category': 62,
+    'pattern-reuse': 58,
+  };
+  return priorities[type] || 40;
+}
+
+function summarizeActionableRemediations(items, limit = 6) {
+  if (!Array.isArray(items)) return [];
+  return items
+    .slice()
+    .sort((left, right) => {
+      const delta = remediationPriority(right && right.type) - remediationPriority(left && left.type);
+      if (delta !== 0) return delta;
+      const rightCount = Number(right && right.evidence && (right.evidence.count || right.evidence.total || right.evidence.highRisk || 0));
+      const leftCount = Number(left && left.evidence && (left.evidence.count || left.evidence.total || left.evidence.highRisk || 0));
+      if (rightCount !== leftCount) return rightCount - leftCount;
+      return String((left && left.target) || '').localeCompare(String((right && right.target) || ''));
+    })
+    .slice(0, limit)
+    .map((item) => ({
+      ...item,
+      priority: remediationPriority(item.type),
+      title: `${String(item.action || 'review').replace(/-/g, ' ')} · ${item.target || 'system'}`,
+      badge: String(item.type || 'remediation').replace(/-/g, ' '),
+    }));
+}
+
+function summarizeMcpServerInventory(projectRoot = PROJECT_ROOT) {
+  const configPath = path.join(projectRoot, '.mcp.json');
+  const parsed = readJsonFile(configPath);
+  const mcpServers = parsed && parsed.mcpServers && typeof parsed.mcpServers === 'object'
+    ? Object.keys(parsed.mcpServers).sort((left, right) => left.localeCompare(right))
+    : [];
+  return {
+    configPath: fs.existsSync(configPath) ? configPath : null,
+    configuredServers: mcpServers,
+    configuredServerCount: mcpServers.length,
+  };
+}
+
+function computeAgentSurfaceInventory(feedbackDir, options = {}) {
+  const readiness = options.readiness || generateAgentReadinessReport({ projectRoot: PROJECT_ROOT });
+  const auditEntries = Array.isArray(options.auditEntries)
+    ? options.auditEntries
+    : readJSONL(path.join(feedbackDir, AUDIT_LOG_FILENAME));
+  const decisionRecords = Array.isArray(options.decisionRecords)
+    ? options.decisionRecords
+    : readDecisionLog(path.join(feedbackDir, DECISION_LOG_FILENAME));
+  const toolBuckets = new Map();
+  const sourceBuckets = new Map();
+  const toolNames = new Set();
+
+  function getToolBucket(toolName) {
+    const key = normalizeText(toolName) || 'unknown';
+    toolNames.add(key);
+    if (!toolBuckets.has(key)) {
+      toolBuckets.set(key, {
+        toolName: key,
+        evaluations: 0,
+        allow: 0,
+        warn: 0,
+        deny: 0,
+        intercepted: 0,
+      });
+    }
+    return toolBuckets.get(key);
+  }
+
+  for (const record of decisionRecords) {
+    if (!record || record.recordType !== 'evaluation') continue;
+    getToolBucket(record.toolName).evaluations += 1;
+  }
+
+  for (const entry of auditEntries) {
+    if (!entry) continue;
+    const bucket = getToolBucket(entry.toolName);
+    if (entry.decision === 'allow') bucket.allow += 1;
+    if (entry.decision === 'warn') {
+      bucket.warn += 1;
+      bucket.intercepted += 1;
+    }
+    if (entry.decision === 'deny') {
+      bucket.deny += 1;
+      bucket.intercepted += 1;
+    }
+    const sourceKey = normalizeText(entry.source) || 'unknown';
+    sourceBuckets.set(sourceKey, (sourceBuckets.get(sourceKey) || 0) + 1);
+  }
+
+  const observedTools = [...toolBuckets.values()]
+    .sort((left, right) => {
+      if (right.intercepted !== left.intercepted) return right.intercepted - left.intercepted;
+      if (right.evaluations !== left.evaluations) return right.evaluations - left.evaluations;
+      return left.toolName.localeCompare(right.toolName);
+    })
+    .slice(0, 8);
+  const policySources = [...sourceBuckets.entries()]
+    .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
+    .map(([source, count]) => ({ source, count }))
+    .slice(0, 6);
+  const mcpInventory = summarizeMcpServerInventory(PROJECT_ROOT);
+
+  return {
+    profile: readiness.permissions.profile,
+    tier: readiness.permissions.tier,
+    configuredServerCount: mcpInventory.configuredServerCount,
+    configuredServers: mcpInventory.configuredServers,
+    observedToolCount: toolNames.size,
+    observedTools,
+    policySources,
+    writeCapableTools: readiness.permissions.writeCapableTools.slice(0, 8),
+    activeBootstrapFiles: readiness.bootstrap.requiredPresent,
+    requiredBootstrapFiles: readiness.bootstrap.requiredCount,
+  };
+}
+
+function computeBackgroundAgentMode(feedbackDir, options = {}) {
+  const governance = generateGovernanceReport({
+    periodHours: options.periodHours || 24,
+    feedbackDir,
+  });
+  const workflowSummary = options.workflowSummary || summarizeWorkflowRuns(feedbackDir);
+  const topRunType = Object.entries(governance.byType || {})
+    .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))[0] || null;
+  const latestRun = workflowSummary.latestRun || null;
+  const checkpointCoverage = safeRate(workflowSummary.reviewedRuns || 0, workflowSummary.proofBackedRuns || 0);
+
+  return {
+    ...governance,
+    checkpointCoverage,
+    reviewedRuns: workflowSummary.reviewedRuns || 0,
+    proofBackedRuns: workflowSummary.proofBackedRuns || 0,
+    latestRun,
+    topRunType: topRunType ? { runType: topRunType[0], count: topRunType[1] } : null,
+    recommendedMode: governance.blocked > 0 || governance.failed > 0
+      ? 'checkpoint-heavy'
+      : governance.total > 0
+        ? 'operator-light'
+        : 'bootstrapping',
+  };
+}
+
+function computeRegulatedBuyerProof(settingsStatus, workflowSummary, decisions, readiness) {
+  const origins = Array.isArray(settingsStatus && settingsStatus.origins) ? settingsStatus.origins : [];
+  const activeLayers = Array.isArray(settingsStatus && settingsStatus.activeLayers)
+    ? settingsStatus.activeLayers.filter((layer) => layer && layer.exists)
+    : [];
+  const latestRun = workflowSummary && workflowSummary.latestRun ? workflowSummary.latestRun : null;
+  const proofArtifacts = latestRun && Array.isArray(latestRun.proofArtifacts) ? latestRun.proofArtifacts : [];
+
+  return {
+    policyOriginCount: origins.length,
+    activeLayerCount: activeLayers.length,
+    reviewedRuns: workflowSummary && workflowSummary.reviewedRuns || 0,
+    proofBackedRuns: workflowSummary && workflowSummary.proofBackedRuns || 0,
+    checkpointCoverage: safeRate(
+      workflowSummary && workflowSummary.reviewedRuns || 0,
+      workflowSummary && workflowSummary.proofBackedRuns || 0
+    ),
+    decisionEvaluations: decisions && decisions.evaluationCount || 0,
+    appendOnlyAuditReady: Boolean(decisions && decisions.evaluationCount > 0),
+    runtimeIsolation: Boolean(readiness && readiness.articleAlignment && readiness.articleAlignment.runtimeIsolation),
+    latestWorkflowName: latestRun ? (latestRun.workflowName || latestRun.workflowId) : null,
+    latestProofArtifacts: proofArtifacts.slice(0, 3),
+    latestPolicyOrigin: origins[0] || null,
+  };
+}
+
 function resolveTeamWindowHours(analyticsWindow) {
   const window = analyticsWindow && analyticsWindow.window;
   if (window === 'today') return 24;
@@ -1416,9 +1599,19 @@ function generateDashboard(feedbackDir, options = {}) {
       availability: 'private_core',
     };
   const readiness = generateAgentReadinessReport({ projectRoot: PROJECT_ROOT });
+  const feedbackAnalysis = analyzeFeedback(path.join(feedbackDir, 'feedback-log.jsonl'));
   const harness = computeHarnessOverview(feedbackDir, entries);
   const interventionPolicy = getInterventionPolicySummary(feedbackDir);
+  const decisionRecords = readDecisionLog(path.join(feedbackDir, DECISION_LOG_FILENAME));
   const decisions = computeDecisionMetrics(feedbackDir);
+  const actionableRemediations = summarizeActionableRemediations(
+    feedbackAnalysis && feedbackAnalysis.actionableRemediations
+  );
+  const agentSurfaceInventory = computeAgentSurfaceInventory(feedbackDir, {
+    readiness,
+    auditEntries,
+    decisionRecords,
+  });
   const settingsStatus = getSettingsStatus({ projectRoot: PROJECT_ROOT });
   settingsStatus.routingPreview = {
     dashboardTool: routeProfile({
@@ -1433,6 +1626,16 @@ function generateDashboard(feedbackDir, options = {}) {
       settingsOptions: { projectRoot: PROJECT_ROOT },
     }),
   };
+  const workflowSummary = summarizeWorkflowRuns(feedbackDir);
+  const backgroundAgents = computeBackgroundAgentMode(feedbackDir, {
+    workflowSummary,
+  });
+  const regulatedProof = computeRegulatedBuyerProof(
+    settingsStatus,
+    workflowSummary,
+    decisions,
+    readiness,
+  );
 
   // Live metrics — gate hit rate, lesson effectiveness, error trend
   const now = Date.now();
@@ -1520,6 +1723,11 @@ function generateDashboard(feedbackDir, options = {}) {
     observability,
     instrumentation,
     readiness,
+    feedbackAnalysis,
+    actionableRemediations,
+    agentSurfaceInventory,
+    backgroundAgents,
+    regulatedProof,
     interventionPolicy,
     decisions,
     settingsStatus,