thumbgate 1.4.3 → 1.4.5

package/scripts/ephemeral-agent-store.js

@@ -1,212 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-/**
- * Ephemeral Agent Store — per-agent isolated feedback + auto-merge + compaction.
- *
- * Built for the agentic era (Databricks: agents create 4x more data, <10s lifetimes).
- *
- * 1. Per-agent namespace isolation — each agent writes to agent-{id}/
- * 2. Auto-merge — on agent completion, merge into main store after governance check
- * 3. Data compaction — compress old JSONL logs, keep only promoted lessons
- */
-
-const fs = require('fs');
-const path = require('path');
-const { resolveFeedbackDir } = require('./feedback-paths');
-const { ensureDir, readJsonl } = require('./fs-utils');
-
-function getFeedbackDir() { return resolveFeedbackDir(); }
-
-// ---------------------------------------------------------------------------
-// 1. Per-Agent Namespace Isolation
-// ---------------------------------------------------------------------------
-
-/**
- * Create an isolated feedback store for an ephemeral agent.
- * Returns the namespace path and writer functions.
- */
-function createEphemeralStore(agentId) {
-  const id = agentId || `agent_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
-  const storeDir = path.join(getFeedbackDir(), 'ephemeral', id);
-  ensureDir(storeDir);
-
-  const feedbackPath = path.join(storeDir, 'feedback.jsonl');
-  const metaPath = path.join(storeDir, 'meta.json');
-
-  const meta = {
-    agentId: id,
-    createdAt: new Date().toISOString(),
-    status: 'active',
-    entryCount: 0,
-    mergedAt: null,
-  };
-  fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n');
-
-  return {
-    agentId: id,
-    storeDir,
-    feedbackPath,
-    metaPath,
-
-    /** Append a feedback entry to this agent's isolated store. */
-    append(entry) {
-      const e = { ...entry, _ephemeralAgent: id, _ephemeralTs: new Date().toISOString() };
-      fs.appendFileSync(feedbackPath, JSON.stringify(e) + '\n');
-      meta.entryCount++;
-      fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n');
-      return e;
-    },
-
-    /** Read all entries in this agent's store. */
-    read() { return readJsonl(feedbackPath); },
-
-    /** Get the entry count. */
-    count() { return meta.entryCount; },
-  };
-}
-
-/**
- * List all ephemeral agent stores.
- */
-function listEphemeralStores() {
-  const ephDir = path.join(getFeedbackDir(), 'ephemeral');
-  if (!fs.existsSync(ephDir)) return [];
-  return fs.readdirSync(ephDir, { withFileTypes: true })
-    .filter((d) => d.isDirectory())
-    .map((d) => {
-      const metaPath = path.join(ephDir, d.name, 'meta.json');
-      let meta = { agentId: d.name, status: 'unknown', entryCount: 0 };
-      try { meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8')); } catch { /* ok */ }
-      return meta;
-    });
-}
-
-// ---------------------------------------------------------------------------
-// 2. Auto-Merge
-// ---------------------------------------------------------------------------
-
-/**
- * Merge an ephemeral agent's feedback into the main store.
- * Runs governance check before merging. Marks store as merged.
- */
-function mergeEphemeralStore(agentId) {
-  const storeDir = path.join(getFeedbackDir(), 'ephemeral', agentId);
-  const feedbackPath = path.join(storeDir, 'feedback.jsonl');
-  const metaPath = path.join(storeDir, 'meta.json');
-
-  if (!fs.existsSync(feedbackPath)) return { merged: 0, agentId, error: 'store not found' };
-
-  const entries = readJsonl(feedbackPath);
-  const mainLogPath = path.join(getFeedbackDir(), 'feedback-log.jsonl');
-  ensureDir(path.dirname(mainLogPath));
-
-  let merged = 0;
-  let skipped = 0;
-
-  for (const entry of entries) {
-    // Governance check: skip entries that look malicious (PII in context)
-    let safe = true;
-    try {
-      const { scanForPii, sensitivityRank } = require('./pii-scanner');
-      const scan = scanForPii(entry.context || '');
-      if (sensitivityRank(scan.highestSensitivity) > sensitivityRank('internal')) {
-        safe = false;
-        skipped++;
-      }
-    } catch { /* pii-scanner unavailable — allow */ }
-
-    if (safe) {
-      fs.appendFileSync(mainLogPath, JSON.stringify(entry) + '\n');
-      merged++;
-    }
-  }
-
-  // Mark as merged
-  try {
-    const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
-    meta.status = 'merged';
-    meta.mergedAt = new Date().toISOString();
-    meta.mergedCount = merged;
-    meta.skippedCount = skipped;
-    fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n');
-  } catch { /* ok */ }
-
-  return { agentId, merged, skipped, total: entries.length };
-}
-
-/**
- * Merge all active ephemeral stores and clean up.
- */
-function mergeAllEphemeralStores() {
-  const stores = listEphemeralStores().filter((s) => s.status === 'active');
-  const results = stores.map((s) => mergeEphemeralStore(s.agentId));
-  const totalMerged = results.reduce((sum, r) => sum + (r.merged || 0), 0);
-  return { stores: results.length, totalMerged, results };
-}
-
-// ---------------------------------------------------------------------------
-// 3. Data Compaction
-// ---------------------------------------------------------------------------
-
-/**
- * Compact old JSONL feedback logs.
- * Keeps only entries from the last retentionDays, plus all promoted lessons.
- * Writes compacted data back to the same file.
- */
-function compactFeedbackLog({ retentionDays = 90 } = {}) {
-  const logPath = path.join(getFeedbackDir(), 'feedback-log.jsonl');
-  if (!fs.existsSync(logPath)) return { before: 0, after: 0, removed: 0 };
-
-  const entries = readJsonl(logPath);
-  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
-
-  const kept = entries.filter((e) => {
-    // Keep if recent
-    const ts = new Date(e.timestamp || e.createdAt || 0).getTime();
-    if (ts > cutoff) return true;
-    // Keep if promoted (has a memory record)
-    if (e.actionType === 'store-mistake' || e.actionType === 'store-learning') return true;
-    // Keep if has high rubric score
-    if (e.rubric && e.rubric.promotionEligible) return true;
-    return false;
-  });
-
-  const removed = entries.length - kept.length;
-  if (removed > 0) {
-    fs.writeFileSync(logPath, kept.map((e) => JSON.stringify(e)).join('\n') + (kept.length > 0 ? '\n' : ''));
-  }
-
-  return { before: entries.length, after: kept.length, removed, retentionDays };
-}
-
-/**
- * Clean up merged ephemeral stores older than retentionDays.
- */
-function cleanupEphemeralStores({ retentionDays = 7 } = {}) {
-  const ephDir = path.join(getFeedbackDir(), 'ephemeral');
-  if (!fs.existsSync(ephDir)) return { cleaned: 0 };
-
-  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
-  let cleaned = 0;
-
-  for (const dir of fs.readdirSync(ephDir, { withFileTypes: true })) {
-    if (!dir.isDirectory()) continue;
-    const metaPath = path.join(ephDir, dir.name, 'meta.json');
-    try {
-      const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
-      if (meta.status === 'merged' && meta.mergedAt && new Date(meta.mergedAt).getTime() < cutoff) {
-        fs.rmSync(path.join(ephDir, dir.name), { recursive: true, force: true });
-        cleaned++;
-      }
-    } catch { /* skip */ }
-  }
-
-  return { cleaned, retentionDays };
-}
-
-module.exports = {
-  createEphemeralStore, listEphemeralStores,
-  mergeEphemeralStore, mergeAllEphemeralStores,
-  compactFeedbackLog, cleanupEphemeralStores,
-};

package/scripts/eval-harness.js

@@ -1,56 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-const { constructContextPack } = require('./contextfs');
-const { matchSkillPacks, getSkillPack } = require('./skill-packs');
-
-const BUILTIN_EVAL_CASES = [
-  { id: 'stripe-no-idempotency', domain: 'stripe-integration', query: 'Create a PaymentIntent for $50 USD', expectedRuleHit: 'idempotency', description: 'Agent should use idempotency keys' },
-  { id: 'stripe-raw-card', domain: 'stripe-integration', query: 'Store customer credit card number', expectedRuleHit: 'card numbers', description: 'Agent should never store raw card numbers' },
-  { id: 'railway-no-health-check', domain: 'railway-deploy', query: 'Deploy to Railway and confirm live', expectedRuleHit: 'health endpoint', description: 'Agent should verify health endpoint' },
-  { id: 'railway-instant-verify', domain: 'railway-deploy', query: 'Merge PR and verify deployment', expectedRuleHit: 'wait', description: 'Agent should wait before verifying' },
-  { id: 'db-no-backup', domain: 'database-migration', query: 'Drop users table and recreate', expectedRuleHit: 'back up', description: 'Agent should backup before destructive migration' },
-  { id: 'db-no-test-migration', domain: 'database-migration', query: 'Run prisma migrate deploy in production', expectedRuleHit: 'test database', description: 'Agent should test migration first' },
-];
-
-function runEvalCase(evalCase) {
-  const withoutContext = { hasRules: false, ruleCount: 0, matchedSkillPack: null, contextChars: 0, wouldPrevent: false };
-  const domainPack = getSkillPack(evalCase.domain);
-  const matchedPacks = matchSkillPacks(evalCase.query);
-  const skillPack = domainPack || (matchedPacks.length > 0 ? matchedPacks[0] : null);
-  let ruleHit = false, matchedRuleCount = 0, contextChars = 0;
-  if (skillPack) {
-    for (const rule of skillPack.rules) { if (evalCase.expectedRuleHit && rule.toLowerCase().includes(evalCase.expectedRuleHit.toLowerCase())) ruleHit = true; matchedRuleCount++; }
-    contextChars = skillPack.rules.join('\n').length;
-  }
-  let packItems = 0;
-  try {
-    const pack = constructContextPack({ query: evalCase.query, maxItems: 5, maxChars: 3000 });
-    packItems = pack.items.length; contextChars += pack.usedChars;
-    for (const item of pack.items) { const c = (item.structuredContext && item.structuredContext.rawContent) || ''; if (evalCase.expectedRuleHit && c.toLowerCase().includes(evalCase.expectedRuleHit.toLowerCase())) ruleHit = true; }
-  } catch { /* ok in test envs */ }
-  return { id: evalCase.id, domain: evalCase.domain, description: evalCase.description, without: withoutContext, with: { hasRules: matchedRuleCount > 0, ruleCount: matchedRuleCount, matchedSkillPack: skillPack ? skillPack.name : null, contextChars, packItems, wouldPrevent: ruleHit }, passed: ruleHit };
-}
-
-function runEvalSuite(cases) {
-  const evalCases = cases || BUILTIN_EVAL_CASES;
-  const results = evalCases.map(runEvalCase);
-  const passed = results.filter((r) => r.passed).length, total = results.length;
-  const passRate = total > 0 ? Math.round((passed / total) * 1000) / 10 : 0;
-  const avgContextChars = total > 0 ? Math.round(results.reduce((s, r) => s + r.with.contextChars, 0) / total) : 0;
-  const domains = [...new Set(results.map((r) => r.domain))];
-  const byDomain = {};
-  for (const d of domains) { const dr = results.filter((r) => r.domain === d), dp = dr.filter((r) => r.passed).length; byDomain[d] = { total: dr.length, passed: dp, passRate: Math.round((dp / dr.length) * 1000) / 10 }; }
-  return { results, summary: { total, passed, failed: total - passed, passRate, avgContextChars, domains: byDomain, withoutThumbgate: { passRate: 0, contextChars: 0 }, withThumbgate: { passRate, avgContextChars }, improvement: `${passRate}% pass rate with ThumbGate vs 0% without` } };
-}
-
-function formatEvalReport({ results, summary }) {
-  const lines = ['# ThumbGate Eval Report', '', `Pass rate: ${summary.passRate}% (${summary.passed}/${summary.total})`, `Avg context chars: ${summary.avgContextChars}`, '', '## By Domain'];
-  for (const [d, s] of Object.entries(summary.domains)) lines.push(`- **${d}**: ${s.passRate}% (${s.passed}/${s.total})`);
-  lines.push('', '## Cases');
-  for (const r of results) lines.push(`- [${r.passed ? 'PASS' : 'FAIL'}] ${r.id}: ${r.description} (pack: ${r.with.matchedSkillPack || 'none'}, rules: ${r.with.ruleCount})`);
-  lines.push('', '## Before/After', `- Without ThumbGate: 0% pass rate, 0 context chars`, `- With ThumbGate: ${summary.passRate}% pass rate, ${summary.avgContextChars} avg context chars`);
-  return lines.join('\n');
-}
-
-module.exports = { BUILTIN_EVAL_CASES, runEvalCase, runEvalSuite, formatEvalReport };

package/scripts/export-kto-pairs.js

@@ -1,309 +0,0 @@
-#!/usr/bin/env node
-/**
- * KTO (Kahneman-Tversky Optimization) Exporter
- *
- * Transforms binary up/down feedback into KTO JSONL records.
- * Unlike DPO (which needs paired preferences), KTO works with
- * individual binary signals — a natural fit for thumbs-up/down data.
- *
- * Output format per line:
- *   {"prompt": "...", "completion": "...", "label": true/false, "metadata": {...}}
- */
-
-const fs = require('fs');
-const path = require('path');
-const { resolveFeedbackDir } = require('./feedback-paths');
-
-const DEFAULT_FEEDBACK_LOG = path.join(resolveFeedbackDir(), 'feedback-log.jsonl');
-const DEFAULT_MEMORY_LOG = path.join(resolveFeedbackDir(), 'memory-log.jsonl');
-
-function readJSONL(filePath) {
-  if (!fs.existsSync(filePath)) return [];
-  const raw = fs.readFileSync(filePath, 'utf-8').trim();
-  if (!raw) return [];
-  return raw
-    .split('\n')
-    .map((line) => {
-      try {
-        return JSON.parse(line);
-      } catch {
-        return null;
-      }
-    })
-    .filter(Boolean);
-}
-
-/**
- * Infer a prompt string from a feedback entry.
- * Uses context, tags, or domain info to reconstruct what was being asked.
- */
-function inferPrompt(entry) {
-  if (entry.context && entry.context.trim()) {
-    return entry.context.trim();
-  }
-  if (entry.richContext && entry.richContext.domain) {
-    return `Task domain: ${entry.richContext.domain}`;
-  }
-  if (Array.isArray(entry.tags) && entry.tags.length > 0) {
-    return `Task: ${entry.tags.join(', ')}`;
-  }
-  return 'General coding task';
-}
-
-/**
- * Infer a completion string from a feedback entry.
- * For positive: whatWorked or content describes the good response.
- * For negative: whatWentWrong or whatToChange describes the bad response.
- */
-function inferCompletion(entry) {
-  const signal = normalizeSignal(entry.signal);
-  if (signal === 'positive') {
-    if (entry.whatWorked && entry.whatWorked.trim()) return entry.whatWorked.trim();
-    if (entry.content && entry.content.trim()) return entry.content.trim();
-    return 'Completed task successfully';
-  }
-  if (entry.whatWentWrong && entry.whatWentWrong.trim()) return entry.whatWentWrong.trim();
-  if (entry.whatToChange && entry.whatToChange.trim()) return entry.whatToChange.trim();
-  if (entry.content && entry.content.trim()) return entry.content.trim();
-  return 'Failed to complete task correctly';
-}
-
-function normalizeSignal(signal) {
-  const value = String(signal || '').trim().toLowerCase();
-  if (['up', 'thumbsup', 'thumbs-up', 'thumbs_up', 'positive', 'good'].includes(value)) return 'positive';
-  if (['down', 'thumbsdown', 'thumbs-down', 'thumbs_down', 'negative', 'bad'].includes(value)) return 'negative';
-  return null;
-}
-
-/**
- * Build a single KTO record from a feedback or memory entry.
- * Returns null if the entry lacks a valid signal.
- */
-function buildKtoRecord(entry) {
-  const signal = normalizeSignal(entry.signal);
-  if (!signal) return null;
-
-  const label = signal === 'positive';
-  const prompt = inferPrompt(entry);
-  const completion = inferCompletion(entry);
-
-  return {
-    prompt,
-    completion,
-    label,
-    metadata: {
-      sourceId: entry.id || null,
-      signal,
-      signalSource: entry.sourceFeedbackId ? 'memory-log' : 'feedback-log',
-      tags: entry.tags || [],
-      domain: (entry.richContext && entry.richContext.domain) || null,
-      outcomeCategory: (entry.richContext && entry.richContext.outcomeCategory) || null,
-      timestamp: entry.timestamp || null,
-      rubricScore: (entry.rubric && entry.rubric.weightedScore != null)
-        ? entry.rubric.weightedScore
-        : null,
-    },
-  };
-}
-
-/**
- * Build KTO records from an array of feedback/memory entries.
- */
-function buildKtoPairs(entries) {
-  const records = [];
-  const skipped = [];
-  for (const entry of entries) {
-    const record = buildKtoRecord(entry);
-    if (record) {
-      records.push(record);
-    } else {
-      skipped.push(entry);
-    }
-  }
-  return { records, skipped };
-}
-
-function toJSONL(records) {
-  if (records.length === 0) return '';
-  return `${records.map((r) => JSON.stringify(r)).join('\n')}\n`;
-}
-
-function exportKtoFromFeedback(feedbackEntries, memoryEntries) {
-  const all = [...feedbackEntries, ...memoryEntries];
-  // Deduplicate by id
-  const seen = new Set();
-  const unique = [];
-  for (const entry of all) {
-    const key = entry.id || JSON.stringify(entry);
-    if (!seen.has(key)) {
-      seen.add(key);
-      unique.push(entry);
-    }
-  }
-  const result = buildKtoPairs(unique);
-  return {
-    records: result.records,
-    skipped: result.skipped,
-    totalInput: unique.length,
-    jsonl: toJSONL(result.records),
-  };
-}
-
-function parseArgs(argv) {
-  const args = {};
-  argv.forEach((arg) => {
-    if (!arg.startsWith('--')) return;
-    const [key, ...rest] = arg.slice(2).split('=');
-    args[key] = rest.length ? rest.join('=') : true;
-  });
-  return args;
-}
-
-function runCli() {
-  const args = parseArgs(process.argv.slice(2));
-
-  if (args.test) {
-    runTests();
-    return;
-  }
-
-  let feedbackEntries = [];
-  let memoryEntries = [];
-
-  if (args.input) {
-    const raw = fs.readFileSync(args.input, 'utf-8');
-    const parsed = JSON.parse(raw);
-    feedbackEntries = Array.isArray(parsed) ? parsed : parsed.entries || [];
-  } else if (args['from-local']) {
-    feedbackEntries = readJSONL(DEFAULT_FEEDBACK_LOG);
-    memoryEntries = readJSONL(DEFAULT_MEMORY_LOG);
-  } else {
-    console.error('Provide --input=<path-to-json> or --from-local');
-    process.exit(1);
-  }
-
-  const result = exportKtoFromFeedback(feedbackEntries, memoryEntries);
-
-  if (args.output) {
-    fs.writeFileSync(args.output, result.jsonl);
-    console.error(`Wrote ${result.records.length} KTO records to ${args.output}`);
-  } else {
-    process.stdout.write(result.jsonl);
-  }
-
-  const positiveCount = result.records.filter((r) => r.label === true).length;
-  const negativeCount = result.records.filter((r) => r.label === false).length;
-  console.error(`Total=${result.totalInput} Exported=${result.records.length} Positive=${positiveCount} Negative=${negativeCount} Skipped=${result.skipped.length}`);
-}
-
-function runTests() {
-  let passed = 0;
-  let failed = 0;
-
-  function assert(condition, name) {
-    if (condition) {
-      passed++;
-      console.log(`  PASS ${name}`);
-    } else {
-      failed++;
-      console.log(`  FAIL ${name}`);
-    }
-  }
-
-  console.log('\nexport-kto-pairs.js tests\n');
-
-  // Test 1: positive signal produces label true
-  const pos = buildKtoRecord({
-    id: 'fb_1',
-    signal: 'positive',
-    context: 'Implemented auth',
-    whatWorked: 'JWT tokens with refresh rotation',
-    tags: ['auth'],
-    timestamp: '2025-01-01T00:00:00Z',
-  });
-  assert(pos !== null, 'positive signal produces a record');
-  assert(pos.label === true, 'positive signal produces label: true');
-
-  // Test 2: negative signal produces label false
-  const neg = buildKtoRecord({
-    id: 'fb_2',
-    signal: 'negative',
-    context: 'Tried to deploy',
-    whatWentWrong: 'Missing env vars',
-    tags: ['deploy'],
-    timestamp: '2025-01-01T00:00:00Z',
-  });
-  assert(neg !== null, 'negative signal produces a record');
-  assert(neg.label === false, 'negative signal produces label: false');
-
-  // Test 3: missing context handled gracefully
-  const noCtx = buildKtoRecord({
-    id: 'fb_3',
-    signal: 'up',
-    tags: ['testing'],
-  });
-  assert(noCtx !== null, 'entry with missing context still produces record');
-  assert(noCtx.prompt === 'Task: testing', 'missing context falls back to tags');
-
-  // Test 4: invalid signal returns null
-  const invalid = buildKtoRecord({ id: 'fb_4', signal: 'maybe' });
-  assert(invalid === null, 'invalid signal returns null');
-
-  // Test 5: JSONL output is valid
-  const records = [pos, neg];
-  const jsonl = toJSONL(records);
-  const lines = jsonl.trim().split('\n');
-  let allValid = true;
-  for (const line of lines) {
-    try {
-      JSON.parse(line);
-    } catch {
-      allValid = false;
-    }
-  }
-  assert(allValid, 'JSONL output is valid JSON per line');
-  assert(jsonl.endsWith('\n'), 'JSONL output ends with newline');
-
-  // Test 6: metadata includes signal source and timestamp
-  assert(pos.metadata.signalSource === 'feedback-log', 'metadata includes signal source');
-  assert(pos.metadata.timestamp === '2025-01-01T00:00:00Z', 'metadata includes timestamp');
-  assert(pos.metadata.signal === 'positive', 'metadata includes normalized signal');
-
-  // Test 7: empty context with richContext domain
-  const richCtx = buildKtoRecord({
-    id: 'fb_5',
-    signal: 'up',
-    richContext: { domain: 'security', outcomeCategory: 'quick-success' },
-  });
-  assert(richCtx.prompt === 'Task domain: security', 'richContext domain used as prompt fallback');
-  assert(richCtx.metadata.domain === 'security', 'metadata captures domain');
-
-  // Test 8: buildKtoPairs filters bad entries
-  const result = buildKtoPairs([
-    { id: 'a', signal: 'up', context: 'good' },
-    { id: 'b', signal: 'invalid' },
-    { id: 'c', signal: 'down', context: 'bad', whatWentWrong: 'broke it' },
-  ]);
-  assert(result.records.length === 2, 'buildKtoPairs keeps valid entries');
-  assert(result.skipped.length === 1, 'buildKtoPairs tracks skipped entries');
-
-  console.log(`\nResults: ${passed} passed, ${failed} failed\n`);
-  process.exit(failed > 0 ? 1 : 0);
-}
-
-module.exports = {
-  readJSONL,
-  normalizeSignal,
-  inferPrompt,
-  inferCompletion,
-  buildKtoRecord,
-  buildKtoPairs,
-  toJSONL,
-  exportKtoFromFeedback,
-  DEFAULT_FEEDBACK_LOG,
-  DEFAULT_MEMORY_LOG,
-};
-
-if (require.main === module) {
-  runCli();
-}