thumbgate 1.4.6 → 1.5.1

package/scripts/decision-trace.js

@@ -0,0 +1,354 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Decision Trace — full observability for gate evaluations.
+ *
+ * Inspired by Ethan Mollick's observation that operators need to *see* what
+ * the agent was thinking when it made a decision. ThumbGate already captures
+ * what was blocked; Decision Trace adds:
+ *
+ *   1. Full audit of every evaluation (passes, blocks, AND near-misses)
+ *   2. Near-miss detection: constraints that almost matched
+ *   3. Session trace summaries: single-glance safety posture view
+ *
+ * Near-miss heuristic: extract literal tokens from a regex deny pattern,
+ * count how many appear in the input. If >50% match but the full regex
+ * doesn't, flag as near-miss.
+ */
+
+const crypto = require('node:crypto');
+const path = require('node:path');
+const { readJsonl, appendJsonl } = require('./fs-utils');
+const { resolveFeedbackDir } = require('./feedback-paths');
+const {
+  evaluateConstraints,
+  evaluateInvariants,
+  loadSpecDir,
+} = require('./spec-gate');
+
+const TRACE_FILE = 'decision-trace.jsonl';
+const NEAR_MISS_THRESHOLD = 0.5;
+
+// ---------------------------------------------------------------------------
+// Near-Miss Detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract literal tokens from a regex pattern.
+ * Strips metacharacters and splits on boundaries to find human-readable tokens.
+ */
+function extractLiteralTokens(pattern) {
+  // Remove common regex metacharacters and quantifiers
+  const cleaned = pattern
+    .replace(/\\[sdwbSDWB]/g, ' ')       // char classes
+    .replace(/[.*+?^${}()|[\]\\]/g, ' ') // metacharacters
+    .replace(/\s+/g, ' ')
+    .trim();
+
+  return cleaned
+    .split(/\s+/)
+    .filter((t) => t.length >= 2) // ignore single chars
+    .map((t) => t.toLowerCase());
+}
+
+/**
+ * Compute near-miss score for a constraint against input text.
+ * Returns { isNearMiss, score, matchedTokens, totalTokens }.
+ */
+function computeNearMiss(constraint, inputText) {
+  const tokens = extractLiteralTokens(constraint.deny);
+  if (tokens.length === 0) {
+    return { isNearMiss: false, score: 0, matchedTokens: 0, totalTokens: 0 };
+  }
+
+  const lower = String(inputText).toLowerCase();
+  let matched = 0;
+  for (const token of tokens) {
+    if (lower.includes(token)) matched++;
+  }
+
+  const score = matched / tokens.length;
+  return {
+    isNearMiss: score >= NEAR_MISS_THRESHOLD && score < 1.0,
+    score: Math.round(score * 100) / 100,
+    matchedTokens: matched,
+    totalTokens: tokens.length,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Trace Evaluation
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the combined input text used for near-miss detection.
+ */
+function buildCombinedInput({ tool, command, content } = {}) {
+  return [command, content, tool].filter(Boolean).join(' ');
+}
+
+/**
+ * Evaluate specs with full trace: passes, blocks, and near-misses.
+ */
+function traceEvaluation(specs, context = {}) {
+  const traceId = `trace_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+  const timestamp = new Date().toISOString();
+  const combinedInput = buildCombinedInput(context);
+  const results = [];
+
+  for (const spec of specs) {
+    const constraintResults = evaluateConstraints(spec, context);
+    const invariantResults = evaluateInvariants(spec, context);
+
+    // Annotate constraint results with near-miss info
+    for (const cr of constraintResults) {
+      const constraint = spec.constraints.find((c) => c.id === cr.constraintId);
+      let nearMiss = { isNearMiss: false, score: 0, matchedTokens: 0, totalTokens: 0 };
+
+      if (cr.passed && constraint) {
+        // Only compute near-miss for constraints that passed (weren't blocked)
+        nearMiss = computeNearMiss(constraint, combinedInput);
+      }
+
+      results.push({
+        ...cr,
+        nearMiss: nearMiss.isNearMiss,
+        nearMissScore: nearMiss.score,
+        nearMissDetail: nearMiss.isNearMiss ? nearMiss : null,
+      });
+    }
+
+    // Invariant results (no near-miss concept for invariants)
+    for (const ir of invariantResults) {
+      results.push({
+        ...ir,
+        nearMiss: false,
+        nearMissScore: 0,
+        nearMissDetail: null,
+      });
+    }
+  }
+
+  const blocked = results.filter((r) => !r.passed);
+  const nearMisses = results.filter((r) => r.nearMiss);
+  const passed = results.filter((r) => r.passed && !r.nearMiss);
+
+  return {
+    traceId,
+    timestamp,
+    allowed: blocked.length === 0,
+    results,
+    blocked,
+    nearMisses,
+    passed,
+    counts: {
+      total: results.length,
+      blocked: blocked.length,
+      nearMiss: nearMisses.length,
+      passed: passed.length,
+    },
+    context: {
+      tool: context.tool || null,
+      command: truncate(context.command, 200),
+      action: truncate(context.action, 200),
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Trace Persistence
+// ---------------------------------------------------------------------------
+
+function getTracePath({ feedbackDir } = {}) {
+  const dir = feedbackDir || resolveFeedbackDir();
+  return path.join(dir, TRACE_FILE);
+}
+
+function recordTrace(trace, options = {}) {
+  const entry = {
+    traceId: trace.traceId,
+    timestamp: trace.timestamp,
+    allowed: trace.allowed,
+    counts: trace.counts,
+    blocked: trace.blocked.map(summarizeResult),
+    nearMisses: trace.nearMisses.map(summarizeResult),
+    context: trace.context,
+  };
+  appendJsonl(getTracePath(options), entry);
+  return entry;
+}
+
+function loadTraces(options = {}) {
+  return readJsonl(getTracePath(options));
+}
+
+function summarizeResult(r) {
+  return {
+    specName: r.specName,
+    id: r.constraintId || r.invariantId,
+    type: r.type,
+    reason: r.reason,
+    severity: r.severity,
+    nearMissScore: r.nearMissScore || 0,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Session Trace Summary
+// ---------------------------------------------------------------------------
+
+/**
+ * Summarize all traces from a session into a single-glance safety posture.
+ */
+function summarizeSessionTraces(traces) {
+  let totalEvaluations = traces.length;
+  let totalChecks = 0;
+  let totalBlocked = 0;
+  let totalNearMisses = 0;
+  let totalPassed = 0;
+
+  const blocksBySpec = new Map();
+  const blocksByConstraint = new Map();
+  const nearMissByConstraint = new Map();
+
+  for (const trace of traces) {
+    const counts = trace.counts || {};
+    totalChecks += counts.total || 0;
+    totalBlocked += counts.blocked || 0;
+    totalNearMisses += counts.nearMiss || 0;
+    totalPassed += counts.passed || 0;
+
+    for (const block of trace.blocked || []) {
+      const specKey = block.specName || 'unknown';
+      blocksBySpec.set(specKey, (blocksBySpec.get(specKey) || 0) + 1);
+      const cKey = block.id || 'unknown';
+      blocksByConstraint.set(cKey, (blocksByConstraint.get(cKey) || 0) + 1);
+    }
+
+    for (const nm of trace.nearMisses || []) {
+      const cKey = nm.id || 'unknown';
+      const existing = nearMissByConstraint.get(cKey) || { count: 0, maxScore: 0 };
+      existing.count += 1;
+      existing.maxScore = Math.max(existing.maxScore, nm.nearMissScore || 0);
+      nearMissByConstraint.set(cKey, existing);
+    }
+  }
+
+  return {
+    totalEvaluations,
+    totalChecks,
+    totalBlocked,
+    totalNearMisses,
+    totalPassed,
+    blockRate: totalChecks > 0 ? Math.round((totalBlocked / totalChecks) * 100) : 0,
+    nearMissRate: totalChecks > 0 ? Math.round((totalNearMisses / totalChecks) * 100) : 0,
+    safetyPosture: computeSafetyPosture(totalBlocked, totalNearMisses, totalChecks),
+    topBlockedSpecs: mapToSorted(blocksBySpec, 'name', 'count'),
+    topBlockedConstraints: mapToSorted(blocksByConstraint, 'id', 'count'),
+    topNearMisses: Array.from(nearMissByConstraint.entries())
+      .sort(([, a], [, b]) => b.count - a.count)
+      .slice(0, 10)
+      .map(([id, data]) => ({ id, count: data.count, maxScore: data.maxScore })),
+  };
+}
+
+/**
+ * Format a trace summary as human-readable text.
+ */
+function formatTraceSummary(summary) {
+  const lines = [];
+  lines.push(`Safety Posture: ${summary.safetyPosture.toUpperCase()}`);
+  lines.push(`Evaluations: ${summary.totalEvaluations} | Checks: ${summary.totalChecks}`);
+  lines.push(`Blocked: ${summary.totalBlocked} (${summary.blockRate}%) | Near-Misses: ${summary.totalNearMisses} (${summary.nearMissRate}%) | Passed: ${summary.totalPassed}`);
+
+  if (summary.topBlockedConstraints.length > 0) {
+    lines.push('');
+    lines.push('Top Blocked:');
+    for (const c of summary.topBlockedConstraints) {
+      lines.push(`  - ${c.id}: ${c.count}x`);
+    }
+  }
+
+  if (summary.topNearMisses.length > 0) {
+    lines.push('');
+    lines.push('Top Near-Misses:');
+    for (const nm of summary.topNearMisses) {
+      lines.push(`  - ${nm.id}: ${nm.count}x (max score: ${nm.maxScore})`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function computeSafetyPosture(blocked, nearMisses, total) {
+  if (total === 0) return 'unknown';
+  if (blocked > 0) return 'critical';
+  if (nearMisses > 0) return 'cautious';
+  return 'clean';
+}
+
+function mapToSorted(map, keyName, valueName) {
+  return Array.from(map.entries())
+    .sort(([, a], [, b]) => b - a)
+    .slice(0, 10)
+    .map(([k, v]) => ({ [keyName]: k, [valueName]: v }));
+}
+
+function truncate(value, maxLength) {
+  if (value === undefined || value === null) return null;
+  const text = String(value).trim();
+  return text ? text.slice(0, maxLength) : null;
+}
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+function isCliInvocation(argv = process.argv) {
+  const invokedPath = argv[1];
+  return invokedPath ? path.resolve(invokedPath) === __filename : false;
+}
+
+if (isCliInvocation()) {
+  const command = process.argv[2] || 'summary';
+
+  if (command === 'summary') {
+    const traces = loadTraces();
+    const summary = summarizeSessionTraces(traces);
+    console.log(formatTraceSummary(summary));
+  } else if (command === 'json') {
+    const traces = loadTraces();
+    const summary = summarizeSessionTraces(traces);
+    console.log(JSON.stringify(summary, null, 2));
+  } else if (command === 'eval') {
+    // Evaluate current specs against a test command
+    const testCommand = process.argv[3] || '';
+    const specs = loadSpecDir();
+    const trace = traceEvaluation(specs, { command: testCommand, action: testCommand });
+    console.log(JSON.stringify({
+      allowed: trace.allowed,
+      counts: trace.counts,
+      blocked: trace.blocked.map(summarizeResult),
+      nearMisses: trace.nearMisses.map(summarizeResult),
+    }, null, 2));
+  } else {
+    console.error(`Unknown command: ${command}. Use: summary, json, eval`);
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  NEAR_MISS_THRESHOLD,
+  computeNearMiss,
+  extractLiteralTokens,
+  formatTraceSummary,
+  loadTraces,
+  recordTrace,
+  summarizeSessionTraces,
+  traceEvaluation,
+};

package/scripts/feedback-loop.js

@@ -1393,8 +1393,6 @@ function analyzeFeedback(logPath) {
   let totalNegative = 0;
 
   for (const entry of entries) {
-    if (isAuditTrailEntry(entry)) continue;
-
     if (entry.signal === 'positive') totalPositive++;
     if (entry.signal === 'negative') totalNegative++;
 
@@ -1428,8 +1426,7 @@ function analyzeFeedback(logPath) {
 
   const total = totalPositive + totalNegative;
   const approvalRate = total > 0 ? Math.round((totalPositive / total) * 1000) / 1000 : 0;
-  const realEntries = entries.filter((entry) => !isAuditTrailEntry(entry));
-  const recent = realEntries.slice(-20);
+  const recent = entries.slice(-20);
   const recentPos = recent.filter((e) => e.signal === 'positive').length;
   const recentRate = recent.length > 0 ? Math.round((recentPos / recent.length) * 1000) / 1000 : 0;
 
@@ -1438,7 +1435,7 @@ function analyzeFeedback(logPath) {
   const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
   const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
   const windowStats = { '7d': { total: 0, positive: 0 }, '30d': { total: 0, positive: 0 } };
-  for (const entry of realEntries) {
+  for (const entry of entries) {
     const ts = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
     const age = now - ts;
     if (age <= SEVEN_DAYS_MS) {
@@ -1701,12 +1698,11 @@ function writePreventionRules(filePath, minOccurrences = 2) {
 function feedbackSummary(recentN = 20, options = {}) {
   const { FEEDBACK_LOG_PATH } = getFeedbackPaths(options);
   const entries = readJSONL(FEEDBACK_LOG_PATH);
-  const realEntries = entries.filter((entry) => !isAuditTrailEntry(entry));
-  if (realEntries.length === 0) {
+  if (entries.length === 0) {
     return '## Feedback Summary\nNo feedback recorded yet.';
   }
 
-  const recent = realEntries.slice(-recentN);
+  const recent = entries.slice(-recentN);
   const positive = recent.filter((e) => e.signal === 'positive').length;
   const negative = recent.filter((e) => e.signal === 'negative').length;
   const pct = Math.round((positive / recent.length) * 100);

package/scripts/rate-limiter.js

@@ -11,23 +11,37 @@ const {
 
 const USAGE_FILE = path.join(process.env.HOME || '/tmp', '.thumbgate', 'usage-limits.json');
 
+// ──────────────────────────────────────────────────────────
+// NEW: Lifetime caps on free tier — users hit the wall fast
+// and must upgrade to keep using core features.
+// ──────────────────────────────────────────────────────────
 const FREE_TIER_LIMITS = {
-  capture_feedback: { daily: 3, label: 'feedback captures' },
-  search_lessons: { daily: 5, label: 'lesson searches' },
-  search_thumbgate: { daily: 5, label: 'ThumbGate searches' },
-  commerce_recall: { daily: 5, label: 'commerce recalls' },
-  export_dpo: { daily: 0, label: 'DPO exports (Pro only)' },
-  export_databricks: { daily: 0, label: 'Databricks exports (Pro only)' },
+  capture_feedback:   { daily: Infinity, lifetime: 3,  label: 'feedback captures' },
+  prevention_rules:   { daily: Infinity, lifetime: 1,  label: 'prevention rules generated' },
+  recall:             { daily: 0,        lifetime: 0,  label: 'recall queries (Pro only)' },
+  search_lessons:     { daily: 0,        lifetime: 0,  label: 'lesson searches (Pro only)' },
+  search_thumbgate:   { daily: 0,        lifetime: 0,  label: 'ThumbGate searches (Pro only)' },
+  commerce_recall:    { daily: 0,        lifetime: 0,  label: 'commerce recalls (Pro only)' },
+  export_dpo:         { daily: 0,        lifetime: 0,  label: 'DPO exports (Pro only)' },
+  export_databricks:  { daily: 0,        lifetime: 0,  label: 'Databricks exports (Pro only)' },
+  construct_context_pack: { daily: Infinity, lifetime: 3, label: 'context packs' },
 };
 
-const FREE_TIER_MAX_GATES = 5;
+const FREE_TIER_MAX_GATES = 1; // Down from 5 — one auto-promoted gate, then paywall
 
-const UPGRADE_MESSAGE = `Pro: ${PRO_PRICE_LABEL} — dashboard and DPO export: ${PRO_MONTHLY_PAYMENT_LINK}\n  Team: ${TEAM_PRICE_LABEL} after workflow qualification.`;
+const UPGRADE_MESSAGE = `Pro: ${PRO_PRICE_LABEL} — unlimited captures, recall, prevention rules, and dashboard: ${PRO_MONTHLY_PAYMENT_LINK}\n  Team: ${TEAM_PRICE_LABEL} after workflow qualification.`;
+
+const PAYWALL_MESSAGES = {
+  capture_feedback: 'You\'ve used all 3 free feedback captures. Your agent is still making mistakes — upgrade to Pro to capture every one and build real prevention rules.',
+  prevention_rules: 'Free tier includes 1 prevention rule. Your agents need more protection — upgrade to Pro for unlimited rules.',
+  recall: 'Recall is a Pro feature. Your past feedback is stored locally — upgrade to search and reuse it.',
+  search_lessons: 'Lesson search is a Pro feature. Upgrade to find patterns in your agent\'s mistakes.',
+  default: 'This feature requires Pro. Start a 7-day free trial — no credit card required.',
+};
 
 function isProTier(authContext) {
   if (authContext && authContext.tier === 'pro') return true;
   if (process.env.THUMBGATE_API_KEY || process.env.THUMBGATE_PRO_MODE === '1' || process.env.THUMBGATE_NO_RATE_LIMIT === '1') return true;
-  // Also check license file for real customer Pro verification
   try {
     const { isProLicensed } = require('./license');
     if (isProLicensed()) return true;
@@ -62,38 +76,79 @@ function todayKey() {
 
 /**
  * Check and increment usage for a given action.
+ * Now enforces LIFETIME limits in addition to daily limits.
  * Returns { allowed: true } or { allowed: false, message: string }
  */
 function checkLimit(action, authContext) {
   if (isProTier(authContext)) return { allowed: true };
 
   const limitEntry = FREE_TIER_LIMITS[action];
-  if (limitEntry == null) return { allowed: true }; // no limit for this action
+  if (limitEntry == null) return { allowed: true };
 
   const dailyLimit = typeof limitEntry === 'object' ? limitEntry.daily : limitEntry;
+  const lifetimeLimit = typeof limitEntry === 'object' ? limitEntry.lifetime : Infinity;
 
   const usage = loadUsage();
   const today = todayKey();
 
-  // Reset if different day
+  // Reset daily counts if different day
   if (usage.date !== today) {
     usage.date = today;
     usage.counts = {};
   }
 
   usage.counts = usage.counts || {};
-  const current = usage.counts[action] || 0;
+  usage.lifetime = usage.lifetime || {};
+
+  const dailyCurrent = usage.counts[action] || 0;
+  const lifetimeCurrent = usage.lifetime[action] || 0;
+
+  // Check lifetime limit first (the hard wall)
+  if (lifetimeLimit !== Infinity && lifetimeCurrent >= lifetimeLimit) {
+    const paywallMsg = PAYWALL_MESSAGES[action] || PAYWALL_MESSAGES.default;
+    return {
+      allowed: false,
+      message: `${paywallMsg}\n\n${UPGRADE_MESSAGE}`,
+      used: lifetimeCurrent,
+      limit: lifetimeLimit,
+      limitType: 'lifetime',
+    };
+  }
 
-  if (current >= dailyLimit) {
-    return { allowed: false, message: `Free tier limit reached. Upgrade to Pro for unlimited: https://thumbgate-production.up.railway.app/pro\n${UPGRADE_MESSAGE}`, used: current, limit: dailyLimit };
+  // Check daily limit
+  if (dailyLimit !== Infinity && dailyCurrent >= dailyLimit) {
+    return {
+      allowed: false,
+      message: `Daily limit reached. ${UPGRADE_MESSAGE}`,
+      used: dailyCurrent,
+      limit: dailyLimit,
+      limitType: 'daily',
+    };
   }
 
-  // Increment
-  usage.counts[action] = current + 1;
+  // Increment both counters
+  usage.counts[action] = dailyCurrent + 1;
+  usage.lifetime[action] = lifetimeCurrent + 1;
   saveUsage(usage);
 
-  const used = current + 1;
-  return { allowed: true, used, limit: dailyLimit, remaining: dailyLimit - used };
+  const remaining = lifetimeLimit === Infinity
+    ? Infinity
+    : lifetimeLimit - (lifetimeCurrent + 1);
+
+  // Warn when approaching limit
+  const warningThreshold = lifetimeLimit <= 3 ? 1 : Math.ceil(lifetimeLimit * 0.2);
+  const isNearLimit = remaining <= warningThreshold && remaining > 0;
+
+  return {
+    allowed: true,
+    used: lifetimeCurrent + 1,
+    limit: lifetimeLimit,
+    remaining,
+    limitType: 'lifetime',
+    warning: isNearLimit
+      ? `${remaining} free ${limitEntry.label} remaining. Upgrade to Pro for unlimited.`
+      : undefined,
+  };
 }
 
 /**
@@ -103,14 +158,11 @@ function getUsage(action, authContext) {
   if (isProTier(authContext)) return { count: 0, limit: Infinity, remaining: Infinity };
 
   const limitEntry = FREE_TIER_LIMITS[action];
-  const dailyLimit = limitEntry == null ? Infinity : (typeof limitEntry === 'object' ? limitEntry.daily : limitEntry);
+  const lifetimeLimit = limitEntry == null ? Infinity : (typeof limitEntry === 'object' ? (limitEntry.lifetime ?? Infinity) : Infinity);
   const usage = loadUsage();
-  const today = todayKey();
-
-  if (usage.date !== today) return { count: 0, limit: dailyLimit, remaining: dailyLimit };
 
-  const count = (usage.counts || {})[action] || 0;
-  return { count, limit: dailyLimit, remaining: Math.max(0, dailyLimit - count) };
+  const lifetimeCount = (usage.lifetime || {})[action] || 0;
+  return { count: lifetimeCount, limit: lifetimeLimit, remaining: Math.max(0, lifetimeLimit - lifetimeCount) };
 }
 
 module.exports = {
@@ -123,5 +175,6 @@ module.exports = {
   FREE_TIER_LIMITS,
   FREE_TIER_MAX_GATES,
   UPGRADE_MESSAGE,
+  PAYWALL_MESSAGES,
   USAGE_FILE,
 };