thumbgate 0.9.9

package/scripts/ephemeral-agent-store.js

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Ephemeral Agent Store — per-agent isolated feedback + auto-merge + compaction.
+ *
+ * Built for the agentic era (Databricks: agents create 4x more data, <10s lifetimes).
+ *
+ * 1. Per-agent namespace isolation — each agent writes to agent-{id}/
+ * 2. Auto-merge — on agent completion, merge into main store after governance check
+ * 3. Data compaction — compress old JSONL logs, keep only promoted lessons
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { resolveFeedbackDir } = require('./feedback-paths');
+
+function getFeedbackDir() { return resolveFeedbackDir(); }
+function ensureDir(p) { if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true }); }
+
+function readJsonl(fp) {
+  if (!fs.existsSync(fp)) return [];
+  const raw = fs.readFileSync(fp, 'utf-8').trim();
+  if (!raw) return [];
+  return raw.split('\n').map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
+}
+
+// ---------------------------------------------------------------------------
+// 1. Per-Agent Namespace Isolation
+// ---------------------------------------------------------------------------
+
+/**
+ * Create an isolated feedback store for an ephemeral agent.
+ * Returns the namespace path and writer functions.
+ */
+function createEphemeralStore(agentId) {
+  const id = agentId || `agent_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+  const storeDir = path.join(getFeedbackDir(), 'ephemeral', id);
+  ensureDir(storeDir);
+
+  const feedbackPath = path.join(storeDir, 'feedback.jsonl');
+  const metaPath = path.join(storeDir, 'meta.json');
+
+  const meta = {
+    agentId: id,
+    createdAt: new Date().toISOString(),
+    status: 'active',
+    entryCount: 0,
+    mergedAt: null,
+  };
+  fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n');
+
+  return {
+    agentId: id,
+    storeDir,
+    feedbackPath,
+    metaPath,
+
+    /** Append a feedback entry to this agent's isolated store. */
+    append(entry) {
+      const e = { ...entry, _ephemeralAgent: id, _ephemeralTs: new Date().toISOString() };
+      fs.appendFileSync(feedbackPath, JSON.stringify(e) + '\n');
+      meta.entryCount++;
+      fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n');
+      return e;
+    },
+
+    /** Read all entries in this agent's store. */
+    read() { return readJsonl(feedbackPath); },
+
+    /** Get the entry count. */
+    count() { return meta.entryCount; },
+  };
+}
+
+/**
+ * List all ephemeral agent stores.
+ */
+function listEphemeralStores() {
+  const ephDir = path.join(getFeedbackDir(), 'ephemeral');
+  if (!fs.existsSync(ephDir)) return [];
+  return fs.readdirSync(ephDir, { withFileTypes: true })
+    .filter((d) => d.isDirectory())
+    .map((d) => {
+      const metaPath = path.join(ephDir, d.name, 'meta.json');
+      let meta = { agentId: d.name, status: 'unknown', entryCount: 0 };
+      try { meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8')); } catch { /* ok */ }
+      return meta;
+    });
+}
+
+// ---------------------------------------------------------------------------
+// 2. Auto-Merge
+// ---------------------------------------------------------------------------
+
+/**
+ * Merge an ephemeral agent's feedback into the main store.
+ * Runs governance check before merging. Marks store as merged.
+ */
+function mergeEphemeralStore(agentId) {
+  const storeDir = path.join(getFeedbackDir(), 'ephemeral', agentId);
+  const feedbackPath = path.join(storeDir, 'feedback.jsonl');
+  const metaPath = path.join(storeDir, 'meta.json');
+
+  if (!fs.existsSync(feedbackPath)) return { merged: 0, agentId, error: 'store not found' };
+
+  const entries = readJsonl(feedbackPath);
+  const mainLogPath = path.join(getFeedbackDir(), 'feedback-log.jsonl');
+  ensureDir(path.dirname(mainLogPath));
+
+  let merged = 0;
+  let skipped = 0;
+
+  for (const entry of entries) {
+    // Governance check: skip entries that look malicious (PII in context)
+    let safe = true;
+    try {
+      const { scanForPii, sensitivityRank } = require('./pii-scanner');
+      const scan = scanForPii(entry.context || '');
+      if (sensitivityRank(scan.highestSensitivity) > sensitivityRank('internal')) {
+        safe = false;
+        skipped++;
+      }
+    } catch { /* pii-scanner unavailable — allow */ }
+
+    if (safe) {
+      fs.appendFileSync(mainLogPath, JSON.stringify(entry) + '\n');
+      merged++;
+    }
+  }
+
+  // Mark as merged
+  try {
+    const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
+    meta.status = 'merged';
+    meta.mergedAt = new Date().toISOString();
+    meta.mergedCount = merged;
+    meta.skippedCount = skipped;
+    fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n');
+  } catch { /* ok */ }
+
+  return { agentId, merged, skipped, total: entries.length };
+}
+
+/**
+ * Merge all active ephemeral stores and clean up.
+ */
+function mergeAllEphemeralStores() {
+  const stores = listEphemeralStores().filter((s) => s.status === 'active');
+  const results = stores.map((s) => mergeEphemeralStore(s.agentId));
+  const totalMerged = results.reduce((sum, r) => sum + (r.merged || 0), 0);
+  return { stores: results.length, totalMerged, results };
+}
+
+// ---------------------------------------------------------------------------
+// 3. Data Compaction
+// ---------------------------------------------------------------------------
+
+/**
+ * Compact old JSONL feedback logs.
+ * Keeps only entries from the last retentionDays, plus all promoted lessons.
+ * Writes compacted data back to the same file.
+ */
+function compactFeedbackLog({ retentionDays = 90 } = {}) {
+  const logPath = path.join(getFeedbackDir(), 'feedback-log.jsonl');
+  if (!fs.existsSync(logPath)) return { before: 0, after: 0, removed: 0 };
+
+  const entries = readJsonl(logPath);
+  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+
+  const kept = entries.filter((e) => {
+    // Keep if recent
+    const ts = new Date(e.timestamp || e.createdAt || 0).getTime();
+    if (ts > cutoff) return true;
+    // Keep if promoted (has a memory record)
+    if (e.actionType === 'store-mistake' || e.actionType === 'store-learning') return true;
+    // Keep if has high rubric score
+    if (e.rubric && e.rubric.promotionEligible) return true;
+    return false;
+  });
+
+  const removed = entries.length - kept.length;
+  if (removed > 0) {
+    fs.writeFileSync(logPath, kept.map((e) => JSON.stringify(e)).join('\n') + (kept.length > 0 ? '\n' : ''));
+  }
+
+  return { before: entries.length, after: kept.length, removed, retentionDays };
+}
+
+/**
+ * Clean up merged ephemeral stores older than retentionDays.
+ */
+function cleanupEphemeralStores({ retentionDays = 7 } = {}) {
+  const ephDir = path.join(getFeedbackDir(), 'ephemeral');
+  if (!fs.existsSync(ephDir)) return { cleaned: 0 };
+
+  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  let cleaned = 0;
+
+  for (const dir of fs.readdirSync(ephDir, { withFileTypes: true })) {
+    if (!dir.isDirectory()) continue;
+    const metaPath = path.join(ephDir, dir.name, 'meta.json');
+    try {
+      const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
+      if (meta.status === 'merged' && meta.mergedAt && new Date(meta.mergedAt).getTime() < cutoff) {
+        fs.rmSync(path.join(ephDir, dir.name), { recursive: true, force: true });
+        cleaned++;
+      }
+    } catch { /* skip */ }
+  }
+
+  return { cleaned, retentionDays };
+}
+
+module.exports = {
+  createEphemeralStore, listEphemeralStores,
+  mergeEphemeralStore, mergeAllEphemeralStores,
+  compactFeedbackLog, cleanupEphemeralStores,
+};

package/scripts/eval-harness.js

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+'use strict';
+
+const { constructContextPack } = require('./contextfs');
+const { matchSkillPacks, getSkillPack } = require('./skill-packs');
+
+const BUILTIN_EVAL_CASES = [
+  { id: 'stripe-no-idempotency', domain: 'stripe-integration', query: 'Create a PaymentIntent for $50 USD', expectedRuleHit: 'idempotency', description: 'Agent should use idempotency keys' },
+  { id: 'stripe-raw-card', domain: 'stripe-integration', query: 'Store customer credit card number', expectedRuleHit: 'card numbers', description: 'Agent should never store raw card numbers' },
+  { id: 'railway-no-health-check', domain: 'railway-deploy', query: 'Deploy to Railway and confirm live', expectedRuleHit: 'health endpoint', description: 'Agent should verify health endpoint' },
+  { id: 'railway-instant-verify', domain: 'railway-deploy', query: 'Merge PR and verify deployment', expectedRuleHit: 'wait', description: 'Agent should wait before verifying' },
+  { id: 'db-no-backup', domain: 'database-migration', query: 'Drop users table and recreate', expectedRuleHit: 'back up', description: 'Agent should backup before destructive migration' },
+  { id: 'db-no-test-migration', domain: 'database-migration', query: 'Run prisma migrate deploy in production', expectedRuleHit: 'test database', description: 'Agent should test migration first' },
+];
+
+function runEvalCase(evalCase) {
+  const withoutContext = { hasRules: false, ruleCount: 0, matchedSkillPack: null, contextChars: 0, wouldPrevent: false };
+  const domainPack = getSkillPack(evalCase.domain);
+  const matchedPacks = matchSkillPacks(evalCase.query);
+  const skillPack = domainPack || (matchedPacks.length > 0 ? matchedPacks[0] : null);
+  let ruleHit = false, matchedRuleCount = 0, contextChars = 0;
+  if (skillPack) {
+    for (const rule of skillPack.rules) { if (evalCase.expectedRuleHit && rule.toLowerCase().includes(evalCase.expectedRuleHit.toLowerCase())) ruleHit = true; matchedRuleCount++; }
+    contextChars = skillPack.rules.join('\n').length;
+  }
+  let packItems = 0;
+  try {
+    const pack = constructContextPack({ query: evalCase.query, maxItems: 5, maxChars: 3000 });
+    packItems = pack.items.length; contextChars += pack.usedChars;
+    for (const item of pack.items) { const c = (item.structuredContext && item.structuredContext.rawContent) || ''; if (evalCase.expectedRuleHit && c.toLowerCase().includes(evalCase.expectedRuleHit.toLowerCase())) ruleHit = true; }
+  } catch { /* ok in test envs */ }
+  return { id: evalCase.id, domain: evalCase.domain, description: evalCase.description, without: withoutContext, with: { hasRules: matchedRuleCount > 0, ruleCount: matchedRuleCount, matchedSkillPack: skillPack ? skillPack.name : null, contextChars, packItems, wouldPrevent: ruleHit }, passed: ruleHit };
+}
+
+function runEvalSuite(cases) {
+  const evalCases = cases || BUILTIN_EVAL_CASES;
+  const results = evalCases.map(runEvalCase);
+  const passed = results.filter((r) => r.passed).length, total = results.length;
+  const passRate = total > 0 ? Math.round((passed / total) * 1000) / 10 : 0;
+  const avgContextChars = total > 0 ? Math.round(results.reduce((s, r) => s + r.with.contextChars, 0) / total) : 0;
+  const domains = [...new Set(results.map((r) => r.domain))];
+  const byDomain = {};
+  for (const d of domains) { const dr = results.filter((r) => r.domain === d), dp = dr.filter((r) => r.passed).length; byDomain[d] = { total: dr.length, passed: dp, passRate: Math.round((dp / dr.length) * 1000) / 10 }; }
+  return { results, summary: { total, passed, failed: total - passed, passRate, avgContextChars, domains: byDomain, withoutThumbgate: { passRate: 0, contextChars: 0 }, withThumbgate: { passRate, avgContextChars }, improvement: `${passRate}% pass rate with ThumbGate vs 0% without` } };
+}
+
+function formatEvalReport({ results, summary }) {
+  const lines = ['# ThumbGate Eval Report', '', `Pass rate: ${summary.passRate}% (${summary.passed}/${summary.total})`, `Avg context chars: ${summary.avgContextChars}`, '', '## By Domain'];
+  for (const [d, s] of Object.entries(summary.domains)) lines.push(`- **${d}**: ${s.passRate}% (${s.passed}/${s.total})`);
+  lines.push('', '## Cases');
+  for (const r of results) lines.push(`- [${r.passed ? 'PASS' : 'FAIL'}] ${r.id}: ${r.description} (pack: ${r.with.matchedSkillPack || 'none'}, rules: ${r.with.ruleCount})`);
+  lines.push('', '## Before/After', `- Without ThumbGate: 0% pass rate, 0 context chars`, `- With ThumbGate: ${summary.passRate}% pass rate, ${summary.avgContextChars} avg context chars`);
+  return lines.join('\n');
+}
+
+module.exports = { BUILTIN_EVAL_CASES, runEvalCase, runEvalSuite, formatEvalReport };

package/scripts/evolution-state.js

@@ -0,0 +1,241 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { resolveFeedbackDir } = require('./feedback-paths');
+
+const DEFAULT_SETTINGS = Object.freeze({
+  half_life_days: 7,
+  decay_floor: 0.01,
+  prevention_min_occurrences: 2,
+  verification_max_retries: 3,
+  dpo_beta: 0.1,
+});
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+function appendJSONL(filePath, record) {
+  ensureDir(path.dirname(filePath));
+  fs.appendFileSync(filePath, `${JSON.stringify(record)}\n`);
+}
+
+function normalizeState(state = {}) {
+  const normalized = {
+    version: Number.isInteger(state.version) ? state.version : 1,
+    updatedAt: typeof state.updatedAt === 'string' ? state.updatedAt : null,
+    acceptedMutations: Array.isArray(state.acceptedMutations) ? state.acceptedMutations : [],
+    settings: {
+      ...DEFAULT_SETTINGS,
+      ...(state.settings || {}),
+    },
+  };
+
+  return normalized;
+}
+
+function getEvolutionPaths(feedbackDir = resolveFeedbackDir()) {
+  return {
+    feedbackDir,
+    statePath: path.join(feedbackDir, 'evolution-state.json'),
+    historyPath: path.join(feedbackDir, 'evolution-history.jsonl'),
+    snapshotsDir: path.join(feedbackDir, 'evolution-snapshots'),
+  };
+}
+
+function readEvolutionState(feedbackDir = resolveFeedbackDir()) {
+  const { statePath } = getEvolutionPaths(feedbackDir);
+  if (!fs.existsSync(statePath)) {
+    return normalizeState();
+  }
+
+  try {
+    return normalizeState(JSON.parse(fs.readFileSync(statePath, 'utf8')));
+  } catch {
+    return normalizeState();
+  }
+}
+
+function writeEvolutionState(state, feedbackDir = resolveFeedbackDir()) {
+  const { statePath } = getEvolutionPaths(feedbackDir);
+  const normalized = normalizeState({
+    ...state,
+    updatedAt: new Date().toISOString(),
+  });
+  ensureDir(path.dirname(statePath));
+  fs.writeFileSync(statePath, `${JSON.stringify(normalized, null, 2)}\n`);
+  return normalized;
+}
+
+function getEffectiveSetting(key, fallback, feedbackDir = resolveFeedbackDir()) {
+  const state = readEvolutionState(feedbackDir);
+  const value = state.settings[key];
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function captureEvolutionSnapshot({
+  label = 'snapshot',
+  reason = 'manual',
+  source = 'workspace-evolver',
+  metadata = {},
+  state = undefined,
+} = {}, feedbackDir = resolveFeedbackDir()) {
+  const paths = getEvolutionPaths(feedbackDir);
+  const snapshotId = `evo_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+  const normalizedState = normalizeState(state || readEvolutionState(feedbackDir));
+  const payload = {
+    snapshotId,
+    label,
+    reason,
+    source,
+    metadata,
+    capturedAt: new Date().toISOString(),
+    state: normalizedState,
+  };
+  const snapshotPath = path.join(paths.snapshotsDir, `${snapshotId}.json`);
+
+  ensureDir(paths.snapshotsDir);
+  fs.writeFileSync(snapshotPath, `${JSON.stringify(payload, null, 2)}\n`);
+
+  return {
+    snapshotId,
+    snapshotPath,
+    payload,
+  };
+}
+
+function applyAcceptedMutation({
+  targetKey,
+  nextValue,
+  experimentId = null,
+  summary = '',
+  metrics = null,
+} = {}, feedbackDir = resolveFeedbackDir()) {
+  if (!targetKey) {
+    throw new Error('applyAcceptedMutation requires targetKey');
+  }
+  if (!Number.isFinite(nextValue)) {
+    throw new Error('applyAcceptedMutation requires a numeric nextValue');
+  }
+
+  const paths = getEvolutionPaths(feedbackDir);
+  const currentState = readEvolutionState(feedbackDir);
+  const previousValue = currentState.settings[targetKey];
+  const rollbackSnapshot = captureEvolutionSnapshot({
+    label: `rollback-${targetKey}`,
+    reason: 'accepted-mutation',
+    source: 'workspace-evolver',
+    metadata: {
+      targetKey,
+      previousValue,
+      nextValue,
+      experimentId,
+    },
+    state: currentState,
+  }, feedbackDir);
+
+  const entry = {
+    id: `mutation_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+    experimentId,
+    targetKey,
+    previousValue,
+    nextValue,
+    summary,
+    metrics,
+    rollbackSnapshotId: rollbackSnapshot.snapshotId,
+    acceptedAt: new Date().toISOString(),
+  };
+
+  const nextState = writeEvolutionState({
+    ...currentState,
+    acceptedMutations: [...currentState.acceptedMutations, entry],
+    settings: {
+      ...currentState.settings,
+      [targetKey]: nextValue,
+    },
+  }, feedbackDir);
+
+  appendJSONL(paths.historyPath, {
+    type: 'mutation_kept',
+    targetKey,
+    previousValue,
+    nextValue,
+    experimentId,
+    rollbackSnapshotId: rollbackSnapshot.snapshotId,
+    timestamp: new Date().toISOString(),
+  });
+
+  return {
+    state: nextState,
+    entry,
+    rollbackSnapshot,
+  };
+}
+
+function restoreEvolutionSnapshot(snapshotId, feedbackDir = resolveFeedbackDir()) {
+  if (!snapshotId) {
+    throw new Error('restoreEvolutionSnapshot requires snapshotId');
+  }
+
+  const paths = getEvolutionPaths(feedbackDir);
+  const snapshotPath = path.join(paths.snapshotsDir, `${snapshotId}.json`);
+  if (!fs.existsSync(snapshotPath)) {
+    throw new Error(`Evolution snapshot not found: ${snapshotId}`);
+  }
+
+  const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf8'));
+  const restoredState = writeEvolutionState(snapshot.state, feedbackDir);
+  appendJSONL(paths.historyPath, {
+    type: 'snapshot_restored',
+    snapshotId,
+    timestamp: new Date().toISOString(),
+  });
+
+  return {
+    snapshotId,
+    snapshotPath,
+    state: restoredState,
+  };
+}
+
+function withTemporaryEvolutionSettings(patch, fn, feedbackDir = resolveFeedbackDir()) {
+  const paths = getEvolutionPaths(feedbackDir);
+  const originalExists = fs.existsSync(paths.statePath);
+  const originalContent = originalExists ? fs.readFileSync(paths.statePath, 'utf8') : null;
+  const currentState = readEvolutionState(feedbackDir);
+
+  writeEvolutionState({
+    ...currentState,
+    settings: {
+      ...currentState.settings,
+      ...(patch || {}),
+    },
+  }, feedbackDir);
+
+  try {
+    return fn();
+  } finally {
+    if (!originalExists) {
+      fs.rmSync(paths.statePath, { force: true });
+    } else {
+      ensureDir(path.dirname(paths.statePath));
+      fs.writeFileSync(paths.statePath, originalContent);
+    }
+  }
+}
+
+module.exports = {
+  DEFAULT_SETTINGS,
+  applyAcceptedMutation,
+  captureEvolutionSnapshot,
+  getEffectiveSetting,
+  getEvolutionPaths,
+  readEvolutionState,
+  resolveFeedbackDir,
+  restoreEvolutionSnapshot,
+  withTemporaryEvolutionSettings,
+  writeEvolutionState,
+};