thumbgate 1.27.4 → 1.27.7

package/scripts/activation-quickstart.js

@@ -0,0 +1,187 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Activation: guided first prevention rule + a live demonstrated block.
+//
+// ~98.5% of `init` users never promote their first rule, so they never see
+// "ThumbGate just blocked a repeat mistake" — the aha moment that drives
+// activation (and, downstream, conversion). `quickstart` walks a new user
+// through capturing one real example, promoting it to a block rule, and then
+// firing that rule against the exact action so they watch it get blocked.
+//
+// SAFETY: this is additive and lives in its own module. `init` is untouched.
+// The interactive flow ONLY runs in a real TTY. Non-interactive / piped / CI
+// invocations print a one-line hint and exit 0 without prompting or hanging.
+// ---------------------------------------------------------------------------
+
+const path = require('node:path');
+
+const PKG_ROOT = path.join(__dirname, '..');
+const PRO_URL = 'https://thumbgate.ai';
+
+// Turn a free-text mistake description into a safe, literal gate pattern. We
+// escape regex metacharacters so arbitrary user input can never produce an
+// invalid or runaway regex inside the gates engine.
+function escapeRegex(text) {
+  return String(text).replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
+}
+
+function pkgVersion() {
+  try {
+    return require(path.join(PKG_ROOT, 'package.json')).version;
+  } catch {
+    // Version is cosmetic in the banner; fall back if package.json is unreadable.
+    return '0.0.0';
+  }
+}
+
+// Core, dependency-injected activation flow so it is testable without a real
+// TTY. Callers provide ask() (returns a Promise<string>), out() (line writer),
+// and isTTY. Returns a structured result describing what was promoted/blocked.
+//
+// Optional injectables (deps) keep the test hermetic and let the CLI pass the
+// real modules:
+//   deps.forcePromote(pattern, action) -> { gateId, totalGates }
+//   deps.runGate(input) -> Promise<rawJsonString>
+//   deps.captureFeedback(params)
+//   deps.trackEvent(name, props)
+async function runActivationFlow({ ask, out, isTTY, deps = {} }) {
+  // Hard safety gate: never prompt or hang in non-interactive contexts.
+  if (!isTTY) {
+    out('thumbgate quickstart is an interactive walkthrough — run it in a terminal.');
+    out('Non-interactive setup: npx thumbgate init   (or: npx thumbgate quick-start)');
+    return { interactive: false, promoted: false, blocked: false };
+  }
+
+  const forcePromote = deps.forcePromote
+    || require(path.join(PKG_ROOT, 'scripts', 'auto-promote-gates')).forcePromote;
+  const runGate = deps.runGate
+    || ((input) => require(path.join(PKG_ROOT, 'scripts', 'gates-engine')).runAsync(input));
+  let captureFeedback = deps.captureFeedback;
+  if (captureFeedback === undefined) {
+    try {
+      ({ captureFeedback } = require(path.join(PKG_ROOT, 'scripts', 'feedback-loop')));
+    } catch {
+      // Feedback capture is a nice-to-have here; the rule + block is the aha.
+      captureFeedback = null;
+    }
+  }
+  const trackEvent = deps.trackEvent || (() => {});
+
+  out('');
+  out(`thumbgate quickstart v${pkgVersion()}`);
+  out('');
+  out("Let's set up your first prevention rule and watch it block a repeat mistake.");
+  out('');
+  out('Think of one thing an AI agent did that you never want it to do again.');
+  out('Examples: "git push --force to main", "rm -rf node_modules", "edit .env directly".');
+  out('');
+
+  let mistake = String(await ask('The mistake to block: ') || '').trim();
+  if (!mistake) {
+    mistake = 'git push --force to main';
+    out(`(using a starter example: ${mistake})`);
+  }
+
+  // 1. Capture it as a real thumbs-down example (best-effort).
+  if (typeof captureFeedback === 'function') {
+    try {
+      captureFeedback({
+        signal: 'down',
+        context: mistake,
+        whatWentWrong: mistake,
+        whatToChange: `Block this action: ${mistake}`,
+        tags: 'quickstart,activation,first-rule',
+        gateAction: 'block',
+      });
+    } catch {
+      // Capture failure should not abort the activation aha.
+    }
+  }
+
+  // 2. Promote it into a hard block rule using the existing primitive.
+  const pattern = escapeRegex(mistake);
+  const promotion = forcePromote(pattern, 'block');
+  out('');
+  out(`  Rule promoted: ${promotion.gateId} [block]`);
+  out(`  Active rules now: ${promotion.totalGates}`);
+
+  // 3. Demonstrate the block. Force strict enforcement for the demo so the
+  //    user's brand-new rule hard-blocks (default posture is warn-by-default).
+  //    We restore the prior value so we never mutate the user's environment.
+  const priorStrict = process.env.THUMBGATE_STRICT_ENFORCEMENT;
+  process.env.THUMBGATE_STRICT_ENFORCEMENT = '1';
+  let verdict;
+  try {
+    const demoInput = { tool_name: 'Bash', tool_input: { command: mistake } };
+    const raw = await runGate(demoInput);
+    verdict = JSON.parse(raw);
+  } finally {
+    if (priorStrict === undefined) delete process.env.THUMBGATE_STRICT_ENFORCEMENT;
+    else process.env.THUMBGATE_STRICT_ENFORCEMENT = priorStrict;
+  }
+
+  const decision = verdict?.decision
+    || verdict?.hookSpecificOutput?.permissionDecision;
+  const blocked = decision === 'block' || decision === 'deny';
+
+  out('');
+  if (blocked) {
+    out('  | ThumbGate just blocked it.');
+    out(`  | Action attempted: ${mistake}`);
+    out('  | Verdict: BLOCKED — the agent cannot repeat this mistake.');
+  } else {
+    // Even in warn posture the action is flagged + logged. Be honest about it.
+    out('  | ThumbGate flagged it.');
+    out(`  | Action attempted: ${mistake}`);
+    out('  | Verdict: flagged and logged. Set THUMBGATE_STRICT_ENFORCEMENT=1 to hard-block.');
+  }
+
+  // 4. Tie the value the user just saw to what Pro keeps working.
+  out('');
+  out('  That rule lives in this project. Pro keeps your rules — and the mistakes');
+  out('  you teach it — synced across every machine and agent runtime you work in,');
+  out(`  so the block you just saw follows your whole team. ${PRO_URL}/pricing`);
+  out('');
+
+  try { trackEvent('cli_quickstart_activated', { command: 'quickstart', blocked }); } catch { /* telemetry best-effort */ }
+
+  return {
+    interactive: true,
+    promoted: true,
+    blocked,
+    gateId: promotion.gateId,
+    pattern,
+    mistake,
+    verdict,
+  };
+}
+
+// CLI entrypoint: wires runActivationFlow to a real readline prompt + stdout.
+function quickstart() {
+  const isTTY = Boolean(process.stdout.isTTY && process.stdin.isTTY);
+  const out = (line = '') => console.log(line);
+
+  const trackEvent = (() => {
+    try { return require(path.join(PKG_ROOT, 'scripts', 'cli-telemetry')).trackEvent; } catch { return () => {}; }
+  })();
+
+  if (!isTTY) {
+    // Mirror the non-interactive branch without opening readline (which would
+    // otherwise keep the process alive waiting for input that never comes).
+    return runActivationFlow({ ask: async () => '', out, isTTY: false, deps: { trackEvent } });
+  }
+
+  const readline = require('node:readline');
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  const ask = (q) => new Promise((resolve) => rl.question(q, resolve));
+
+  return runActivationFlow({ ask, out, isTTY: true, deps: { trackEvent } })
+    .catch((err) => {
+      console.error(`quickstart error: ${err?.message ?? err}`);
+      process.exitCode = 1;
+    })
+    .finally(() => rl.close());
+}
+
+module.exports = { runActivationFlow, quickstart, escapeRegex };

package/scripts/agent-memory-lifecycle.js

@@ -2,6 +2,40 @@
 'use strict';
 
 const MEMORY_TYPES = new Set(['episodic', 'semantic', 'procedural', 'preference', 'working']);
+const MEMORY_SCOPES = new Set(['task', 'session', 'user', 'project', 'org']);
+const HIGH_RISK_TERMS = new Set([
+  'billing',
+  'checkout',
+  'compliance',
+  'credential',
+  'data-loss',
+  'deploy',
+  'deployment',
+  'git',
+  'payment',
+  'production',
+  'release',
+  'secret',
+  'security',
+  'stripe',
+  'verification',
+]);
+const KNOWN_ENTITY_PATTERNS = [
+  ['Claude Code', /\bclaude\s+code\b/i, 'agent'],
+  ['Codex', /\bcodex\b/i, 'agent'],
+  ['Cursor', /\bcursor\b/i, 'agent'],
+  ['Gemini CLI', /\bgemini\s+cli\b/i, 'agent'],
+  ['MCP', /\bmcp\b/i, 'protocol'],
+  ['Stripe', /\bstripe\b/i, 'service'],
+  ['GitHub', /\bgithub\b|\bgh\s+/i, 'service'],
+  ['Railway', /\brailway\b/i, 'service'],
+  ['Plausible', /\bplausible\b/i, 'service'],
+  ['PostHog', /\bposthog\b/i, 'service'],
+  ['SQLite', /\bsqlite\b|\bfts5\b/i, 'storage'],
+  ['LanceDB', /\blancedb\b/i, 'storage'],
+  ['Docker', /\bdocker\b/i, 'runtime'],
+  ['npm', /\bnpm\b|\bnpx\b/i, 'runtime'],
+];
 
 function normalizeText(value) {
   if (value === undefined || value === null) return '';
@@ -13,6 +47,178 @@ function normalizeMemoryType(value) {
   return MEMORY_TYPES.has(normalized) ? normalized : 'episodic';
 }
 
+function tokenize(value) {
+  return normalizeText(value)
+    .toLowerCase()
+    .split(/[^a-z0-9_.:/-]+/)
+    .filter(Boolean);
+}
+
+function uniqueByName(entities) {
+  const seen = new Set();
+  return entities.filter((entity) => {
+    const key = normalizeText(entity.name).toLowerCase();
+    if (!key || seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function collectMemoryText(memory = {}) {
+  return [
+    memory.title,
+    memory.content,
+    memory.context,
+    memory.whatWentWrong,
+    memory.whatToChange,
+    memory.whatWorked,
+    memory.domain,
+    memory.skill,
+    Array.isArray(memory.tags) ? memory.tags.join(' ') : memory.tags,
+  ].filter(Boolean).join(' ');
+}
+
+function extractMemoryEntities(memory = {}) {
+  const text = collectMemoryText(memory);
+  const entities = [];
+
+  for (const [name, pattern, type] of KNOWN_ENTITY_PATTERNS) {
+    if (pattern.test(text)) entities.push({ name, type });
+  }
+
+  const commandMatches = text.match(/`([^`]+)`/g) || [];
+  for (const match of commandMatches) {
+    const command = match.slice(1, -1).trim();
+    if (/^(git|npm|npx|node|gh|curl|docker|python|pytest|stripe)\b/i.test(command)) {
+      entities.push({ name: command, type: 'command' });
+    } else if (/[./-]/.test(command)) {
+      entities.push({ name: command, type: 'path' });
+    }
+  }
+
+  const pathMatches = text.match(/\b(?:[a-z0-9_-]+\/)+[a-z0-9_.-]+\b/gi) || [];
+  for (const filePath of pathMatches.slice(0, 8)) {
+    entities.push({ name: filePath, type: 'path' });
+  }
+
+  return uniqueByName(entities).slice(0, 16);
+}
+
+function inferMemoryScope(memory = {}) {
+  const explicit = normalizeText(memory.scope || memory.memoryScope).toLowerCase();
+  if (MEMORY_SCOPES.has(explicit)) return explicit;
+
+  const text = collectMemoryText(memory).toLowerCase();
+  const tags = new Set(Array.isArray(memory.tags) ? memory.tags.map((tag) => normalizeText(tag).toLowerCase()) : []);
+
+  if (tags.has('preference') || /\b(prefer|style|tone|my preference|user preference)\b/.test(text)) return 'user';
+  if (tags.has('org') || tags.has('team') || /\b(enterprise|seat|team|shared|org|compliance|policy|approval)\b/.test(text)) return 'org';
+  if (tags.has('repo') || tags.has('project') || tags.has('release') || tags.has('deployment')
+    || /\b(repo|repository|branch|ci|pull request|github|deploy|production|release|publish)\b/.test(text)) return 'project';
+  if (tags.has('session') || /\b(this session|current session|today|right now)\b/.test(text)) return 'session';
+  return 'task';
+}
+
+function scoreMemoryDecay(memory = {}, options = {}) {
+  const nowMs = options.now ? new Date(options.now).getTime() : Date.now();
+  const timestampMs = memory.timestamp ? new Date(memory.timestamp).getTime() : NaN;
+  const ageDays = Number.isFinite(timestampMs)
+    ? Math.max(0, (nowMs - timestampMs) / (1000 * 60 * 60 * 24))
+    : null;
+  const textTokens = new Set(tokenize(collectMemoryText(memory)));
+  const tags = Array.isArray(memory.tags) ? memory.tags.map((tag) => normalizeText(tag).toLowerCase()) : [];
+  const highRisk = tags.some((tag) => HIGH_RISK_TERMS.has(tag))
+    || [...textTokens].some((token) => HIGH_RISK_TERMS.has(token))
+    || ['critical', 'high'].includes(normalizeText(memory.importance).toLowerCase());
+
+  if (highRisk) {
+    return {
+      state: 'sticky',
+      ageDays,
+      score: 1,
+      reason: 'high-risk memories stay retrievable until explicitly retired',
+    };
+  }
+  if (ageDays === null) {
+    return {
+      state: 'review',
+      ageDays,
+      score: 0.6,
+      reason: 'memory has no timestamp, so it needs review before durable promotion',
+    };
+  }
+  if (ageDays > 180) {
+    return {
+      state: 'archive_candidate',
+      ageDays,
+      score: 0.2,
+      reason: 'old low-risk memory should be consolidated or archived',
+    };
+  }
+  if (ageDays > 60) {
+    return {
+      state: 'review',
+      ageDays,
+      score: 0.55,
+      reason: 'older low-risk memory should be refreshed before it dominates recall',
+    };
+  }
+  return {
+    state: 'active',
+    ageDays,
+    score: 0.85,
+    reason: 'recent memory remains eligible for recall',
+  };
+}
+
+function scoreHybridMemoryMatch(query, memory = {}, options = {}) {
+  const queryTokens = new Set(tokenize(query));
+  const memoryTokens = new Set(tokenize(collectMemoryText(memory)));
+  const queryText = normalizeText(query).toLowerCase();
+  const memoryText = collectMemoryText(memory).toLowerCase();
+  const memoryEntities = extractMemoryEntities(memory);
+  const queryEntityNames = extractMemoryEntities({ content: query }).map((entity) => entity.name.toLowerCase());
+
+  let lexicalMatches = 0;
+  for (const token of queryTokens) {
+    if (memoryTokens.has(token)) lexicalMatches++;
+  }
+  const lexicalScore = queryTokens.size > 0 ? lexicalMatches / queryTokens.size : 0;
+  const phraseScore = queryText && memoryText.includes(queryText) ? 0.35 : 0;
+  const entityMatches = memoryEntities.filter((entity) => queryEntityNames.includes(entity.name.toLowerCase()));
+  const entityScore = queryEntityNames.length > 0 ? entityMatches.length / queryEntityNames.length : 0;
+  const decay = scoreMemoryDecay(memory, options);
+  const lifecycleScore = decay.state === 'archive_candidate' ? -0.15 : decay.state === 'sticky' ? 0.12 : 0;
+  const score = lexicalScore + phraseScore + (entityScore * 0.45) + lifecycleScore;
+
+  return {
+    score: Number(Math.max(0, score).toFixed(4)),
+    lexicalScore: Number(lexicalScore.toFixed(4)),
+    entityScore: Number(entityScore.toFixed(4)),
+    matchedEntities: entityMatches,
+    decayState: decay.state,
+  };
+}
+
+function buildMemoryLifecycleView(memory = {}, options = {}) {
+  const scope = inferMemoryScope(memory);
+  const entities = extractMemoryEntities(memory);
+  const decay = scoreMemoryDecay(memory, options);
+  const retrieval = scoreHybridMemoryMatch(options.query || '', memory, options);
+
+  return {
+    scope,
+    entities,
+    decay,
+    retrievalHints: {
+      hybridScore: retrieval.score,
+      lexicalScore: retrieval.lexicalScore,
+      entityScore: retrieval.entityScore,
+      matchedEntities: retrieval.matchedEntities,
+    },
+  };
+}
+
 function buildMemoryLifecyclePolicy(input = {}) {
   return {
     generatedAt: normalizeText(input.generatedAt) || new Date().toISOString(),
@@ -91,6 +297,11 @@ function evaluateMemoryPromotion(memory = {}, policy = buildMemoryLifecyclePolic
 
 module.exports = {
   buildMemoryLifecyclePolicy,
+  buildMemoryLifecycleView,
   evaluateMemoryPromotion,
+  extractMemoryEntities,
+  inferMemoryScope,
   normalizeMemoryType,
+  scoreHybridMemoryMatch,
+  scoreMemoryDecay,
 };

package/scripts/async-eval-observability.js

@@ -0,0 +1,236 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const DEFAULT_THRESHOLDS = {
+  faithfulness: 0.72,
+  answerRelevance: 0.45,
+  contextPrecision: 0.5,
+};
+
+function tokenize(value) {
+  return String(value || '')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((token) => token.length > 2);
+}
+
+function unique(values) {
+  return [...new Set(values.filter(Boolean))];
+}
+
+function overlapScore(left, right) {
+  const leftTokens = unique(tokenize(left));
+  const rightSet = new Set(tokenize(right));
+  if (leftTokens.length === 0) return 0;
+  const matches = leftTokens.filter((token) => rightSet.has(token)).length;
+  return matches / leftTokens.length;
+}
+
+function splitClaims(response) {
+  return String(response || '')
+    .split(/(?:[.!?]\s+|\n+)/)
+    .map((claim) => claim.trim())
+    .filter((claim) => claim.length > 0);
+}
+
+function normalizeContexts(contexts) {
+  if (Array.isArray(contexts)) return contexts.map(String).filter(Boolean);
+  if (contexts) return [String(contexts)];
+  return [];
+}
+
+function scoreFaithfulness(response, contexts) {
+  const claims = splitClaims(response);
+  const contextText = normalizeContexts(contexts).join('\n');
+  if (claims.length === 0) return { score: 0, supportedClaims: 0, totalClaims: 0 };
+  const supportedClaims = claims.filter((claim) => {
+    const normalized = claim.toLowerCase();
+    return contextText.toLowerCase().includes(normalized) || overlapScore(claim, contextText) >= 0.58;
+  }).length;
+  return {
+    score: Number((supportedClaims / claims.length).toFixed(4)),
+    supportedClaims,
+    totalClaims: claims.length,
+  };
+}
+
+function scoreAnswerRelevance(question, response) {
+  const score = overlapScore(question, response);
+  return {
+    score: Number(score.toFixed(4)),
+    matchedQuestionTerms: unique(tokenize(question).filter((token) => tokenize(response).includes(token))),
+  };
+}
+
+function scoreContextPrecision(question, contexts, reference = '') {
+  const normalizedContexts = normalizeContexts(contexts);
+  const target = [question, reference].filter(Boolean).join('\n');
+  if (normalizedContexts.length === 0) return { score: 0, relevantContexts: 0, totalContexts: 0 };
+
+  let precisionSum = 0;
+  let relevantContexts = 0;
+  normalizedContexts.forEach((context, index) => {
+    const relevant = overlapScore(target, context) >= 0.22 || overlapScore(context, target) >= 0.22;
+    if (relevant) relevantContexts += 1;
+    const precisionAtK = relevantContexts / (index + 1);
+    if (relevant) precisionSum += precisionAtK;
+  });
+
+  const score = relevantContexts === 0 ? 0 : precisionSum / relevantContexts;
+  return {
+    score: Number(score.toFixed(4)),
+    relevantContexts,
+    totalContexts: normalizedContexts.length,
+  };
+}
+
+function evaluateGeneration(testCase, options = {}) {
+  const thresholds = { ...DEFAULT_THRESHOLDS, ...(options.thresholds || {}) };
+  const contexts = normalizeContexts(testCase.retrievedContexts || testCase.contexts || testCase.retrieved_contexts);
+  const faithfulness = scoreFaithfulness(testCase.response || testCase.answer, contexts);
+  const answerRelevance = scoreAnswerRelevance(testCase.question || testCase.user_input, testCase.response || testCase.answer);
+  const contextPrecision = scoreContextPrecision(
+    testCase.question || testCase.user_input,
+    contexts,
+    testCase.reference || testCase.groundTruth || ''
+  );
+  const scores = {
+    faithfulness: faithfulness.score,
+    answerRelevance: answerRelevance.score,
+    contextPrecision: contextPrecision.score,
+  };
+  const passed = scores.faithfulness >= thresholds.faithfulness
+    && scores.answerRelevance >= thresholds.answerRelevance
+    && scores.contextPrecision >= thresholds.contextPrecision;
+
+  return {
+    id: String(testCase.id || testCase.traceId || 'case'),
+    traceId: String(testCase.traceId || testCase.id || ''),
+    passed,
+    scores,
+    thresholds,
+    details: {
+      faithfulness,
+      answerRelevance,
+      contextPrecision,
+    },
+  };
+}
+
+function buildRagasCompatibleRows(cases) {
+  return cases.map((testCase) => ({
+    user_input: testCase.question || testCase.user_input || '',
+    response: testCase.response || testCase.answer || '',
+    retrieved_contexts: normalizeContexts(testCase.retrievedContexts || testCase.contexts || testCase.retrieved_contexts),
+    reference: testCase.reference || testCase.groundTruth || '',
+  }));
+}
+
+function buildLangSmithCompatibleRuns(cases, results) {
+  return cases.map((testCase, index) => ({
+    id: testCase.traceId || testCase.id || `case-${index + 1}`,
+    name: 'thumbgate_async_rag_eval',
+    inputs: { question: testCase.question || testCase.user_input || '' },
+    outputs: { response: testCase.response || testCase.answer || '' },
+    metadata: {
+      evaluator: 'thumbgate-async-eval-observability',
+      caseId: testCase.id || null,
+    },
+    feedback: Object.entries(results[index].scores).map(([key, score]) => ({
+      key,
+      score,
+    })),
+  }));
+}
+
+function buildEvalReport(cases, options = {}) {
+  const normalizedCases = Array.isArray(cases) ? cases : [];
+  const results = normalizedCases.map((testCase) => evaluateGeneration(testCase, options));
+  const passed = results.filter((result) => result.passed).length;
+  const failed = results.length - passed;
+  const aggregate = {
+    faithfulness: average(results.map((result) => result.scores.faithfulness)),
+    answerRelevance: average(results.map((result) => result.scores.answerRelevance)),
+    contextPrecision: average(results.map((result) => result.scores.contextPrecision)),
+  };
+
+  return {
+    generatedAt: new Date().toISOString(),
+    mode: 'async-post-generation',
+    total: results.length,
+    passed,
+    failed,
+    passRate: results.length === 0 ? 0 : Number(((passed / results.length) * 100).toFixed(2)),
+    aggregate,
+    passedThreshold: failed === 0,
+    metrics: ['faithfulness', 'answerRelevance', 'contextPrecision'],
+    sinks: {
+      ci: true,
+      langsmithCompatible: true,
+      ragasCompatible: true,
+    },
+    results,
+    ragasDataset: buildRagasCompatibleRows(normalizedCases),
+    langsmithRuns: buildLangSmithCompatibleRuns(normalizedCases, results),
+  };
+}
+
+function average(values) {
+  const numeric = values.filter((value) => Number.isFinite(value));
+  if (numeric.length === 0) return 0;
+  return Number((numeric.reduce((sum, value) => sum + value, 0) / numeric.length).toFixed(4));
+}
+
+async function runAsyncEvaluation(cases, options = {}) {
+  const report = await new Promise((resolve) => {
+    setImmediate(() => resolve(buildEvalReport(cases, options)));
+  });
+  if (options.outputPath) {
+    fs.mkdirSync(path.dirname(options.outputPath), { recursive: true });
+    fs.writeFileSync(options.outputPath, `${JSON.stringify(report, null, 2)}\n`);
+  }
+  return report;
+}
+
+function loadCases(inputPath) {
+  const payload = JSON.parse(fs.readFileSync(inputPath, 'utf8'));
+  return Array.isArray(payload) ? payload : payload.cases || [];
+}
+
+async function main(argv = process.argv.slice(2)) {
+  const inputIndex = argv.indexOf('--input');
+  const outputIndex = argv.indexOf('--output');
+  const inputPath = inputIndex >= 0 ? argv[inputIndex + 1] : 'bench/observability-eval-suite.json';
+  const outputPath = outputIndex >= 0 ? argv[outputIndex + 1] : 'proof/async-eval-observability-report.json';
+  const report = await runAsyncEvaluation(loadCases(inputPath), { outputPath });
+  process.stdout.write(`${JSON.stringify({
+    outputPath,
+    total: report.total,
+    passed: report.passed,
+    failed: report.failed,
+    passRate: report.passRate,
+  }, null, 2)}\n`);
+  if (!report.passedThreshold) process.exitCode = 1;
+}
+
+module.exports = {
+  DEFAULT_THRESHOLDS,
+  buildEvalReport,
+  buildLangSmithCompatibleRuns,
+  buildRagasCompatibleRows,
+  evaluateGeneration,
+  runAsyncEvaluation,
+  scoreAnswerRelevance,
+  scoreContextPrecision,
+  scoreFaithfulness,
+};
+
+if (require.main === module) {
+  main().catch((err) => {
+    console.error(err.stack || err.message);
+    process.exitCode = 1;
+  });
+}