thumbgate 1.16.13 → 1.16.19

package/scripts/proxy-pointer-rag-guardrails.js

@@ -0,0 +1,189 @@
+#!/usr/bin/env node
+'use strict';
+
+const path = require('path');
+const { listGateTemplates } = require('./gate-templates');
+
+const DOCUMENT_RAG_CATEGORY = 'Document RAG Safety';
+
+function splitCsv(value) {
+  if (Array.isArray(value)) return value.map(String).map((item) => item.trim()).filter(Boolean);
+  if (value === undefined || value === null || value === true) return [];
+  return String(value)
+    .split(',')
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+function unique(values) {
+  return Array.from(new Set(values.filter(Boolean)));
+}
+
+function normalizeBoolean(value) {
+  if (value === true) return true;
+  if (value === false || value === undefined || value === null) return false;
+  return /^(1|true|yes|on)$/i.test(String(value).trim());
+}
+
+function normalizeOptions(options = {}) {
+  const sectionIds = unique([
+    ...splitCsv(options.sections),
+    ...splitCsv(options['section-ids']),
+  ]);
+  const imagePointers = unique([
+    ...splitCsv(options['image-pointers']),
+    ...splitCsv(options.images),
+    ...splitCsv(options.figures),
+  ]);
+  const documentIds = unique([
+    ...splitCsv(options.documents),
+    ...splitCsv(options['document-ids']),
+  ]);
+  const candidateImages = Number.isFinite(Number(options['candidate-images']))
+    ? Number(options['candidate-images'])
+    : null;
+
+  return {
+    ragTool: String(options['rag-tool'] || options.tool || 'proxy-pointer-rag').trim() || 'proxy-pointer-rag',
+    treePath: options['tree-path'] ? path.normalize(String(options['tree-path']).trim()) : null,
+    sectionIds,
+    imagePointers,
+    documentIds,
+    candidateImages,
+    crossDocumentPolicy: String(options['cross-doc-policy'] || options['cross-document-policy'] || '').trim().toLowerCase(),
+    visionFilter: normalizeBoolean(options['vision-filter']),
+    visualClaims: normalizeBoolean(options['visual-claims']),
+  };
+}
+
+function gateApplicability(template, options) {
+  if (template.id === 'require-section-tree-before-multimodal-answer') {
+    return Boolean(options.treePath || options.sectionIds.length > 0);
+  }
+  if (template.id === 'require-image-pointer-grounding') {
+    return options.imagePointers.length > 0;
+  }
+  if (template.id === 'block-cross-document-image-leakage') {
+    return options.documentIds.length > 1 || options.crossDocumentPolicy === 'strict';
+  }
+  if (template.id === 'checkpoint-vision-filter-for-visual-claims') {
+    return options.visualClaims || options.visionFilter || (options.candidateImages !== null && options.candidateImages > 0);
+  }
+  return false;
+}
+
+function buildSignalSummary(options) {
+  const signals = [];
+  if (options.treePath || options.sectionIds.length > 0) {
+    signals.push({
+      id: 'section_tree',
+      label: 'Hierarchical section tree',
+      values: unique([options.treePath, ...options.sectionIds]),
+      risk: 'visual answers should be grounded in document structure, not sliding-window chunks',
+    });
+  }
+  if (options.imagePointers.length > 0) {
+    signals.push({
+      id: 'image_pointers',
+      label: 'Image pointers',
+      values: options.imagePointers,
+      risk: 'every selected visual needs source document, parent section, and path metadata',
+    });
+  }
+  if (options.documentIds.length > 1 || options.crossDocumentPolicy === 'strict') {
+    signals.push({
+      id: 'cross_document_leakage',
+      label: 'Cross-document leakage risk',
+      values: options.documentIds.length > 0 ? options.documentIds : ['strict cross-document policy'],
+      risk: 'a plausible image from the wrong document can invalidate the answer',
+    });
+  }
+  if (options.visualClaims || options.visionFilter || (options.candidateImages !== null && options.candidateImages > 0)) {
+    signals.push({
+      id: 'visual_claims',
+      label: 'Visual claim checkpoint',
+      values: unique([
+        options.visualClaims ? 'visual claims enabled' : null,
+        options.visionFilter ? 'vision filter enabled' : null,
+        options.candidateImages !== null ? `${options.candidateImages} candidate images` : null,
+      ]),
+      risk: 'answers that describe image content may need a vision-model sanity check',
+    });
+  }
+  return signals;
+}
+
+function buildProxyPointerRagGuardrailsPlan(rawOptions = {}, templatesPath) {
+  const options = normalizeOptions(rawOptions);
+  const templates = listGateTemplates(templatesPath)
+    .filter((template) => template.category === DOCUMENT_RAG_CATEGORY);
+  const signals = buildSignalSummary(options);
+  const recommendedTemplates = templates.map((template) => ({
+    ...template,
+    recommended: gateApplicability(template, options),
+  }));
+  const activeTemplates = recommendedTemplates.filter((template) => template.recommended);
+
+  return {
+    name: 'thumbgate-proxy-pointer-rag-guardrails',
+    status: activeTemplates.length > 0 ? 'actionable' : 'ready',
+    ragTool: options.ragTool,
+    treePath: options.treePath,
+    summary: {
+      signalCount: signals.length,
+      templateCount: templates.length,
+      recommendedTemplateCount: activeTemplates.length,
+      candidateImages: options.candidateImages,
+    },
+    signals,
+    templates: recommendedTemplates,
+    nextActions: [
+      'Preserve document hierarchy, section IDs, and image file paths during ingestion.',
+      'Pass section-tree and image-pointer metadata into the agent before it answers with visuals.',
+      'Enable the recommended Document RAG Safety templates as pre-action gates.',
+      'Use a vision filter only for high-impact answers that make claims about visual content.',
+    ],
+    exampleCommand: 'npx thumbgate proxy-pointer-rag-guardrails --tree-path=.rag/tree.json --image-pointers=paper-1/figures/fig2.png --documents=paper-1 --visual-claims --json',
+  };
+}
+
+function formatProxyPointerRagGuardrailsPlan(report) {
+  const lines = [
+    '',
+    'ThumbGate Proxy-Pointer RAG Guardrails',
+    '-'.repeat(42),
+    `Status   : ${report.status}`,
+    `RAG tool : ${report.ragTool}`,
+  ];
+  if (report.treePath) lines.push(`Tree path: ${report.treePath}`);
+  lines.push(`Signals  : ${report.summary.signalCount}`);
+  lines.push(`Templates: ${report.summary.recommendedTemplateCount}/${report.summary.templateCount} recommended`);
+
+  if (report.signals.length > 0) {
+    lines.push('', 'Detected document/RAG risk signals:');
+    for (const signal of report.signals) {
+      lines.push(`  - ${signal.label}: ${signal.values.join(', ')}`);
+      lines.push(`    Risk: ${signal.risk}`);
+    }
+  }
+
+  lines.push('', 'Recommended templates:');
+  for (const template of report.templates.filter((entry) => entry.recommended)) {
+    lines.push(`  - ${template.id} [${template.defaultAction}]`);
+    lines.push(`    ${template.roi}`);
+  }
+  if (report.summary.recommendedTemplateCount === 0) {
+    lines.push('  - No document/RAG signals were passed. Start with --tree-path, --image-pointers, --documents, or --visual-claims.');
+  }
+
+  lines.push('', 'Next actions:');
+  for (const action of report.nextActions) lines.push(`  - ${action}`);
+  lines.push('', `Example: ${report.exampleCommand}`, '');
+  return `${lines.join('\n')}\n`;
+}
+
+module.exports = {
+  buildProxyPointerRagGuardrailsPlan,
+  formatProxyPointerRagGuardrailsPlan,
+  normalizeOptions,
+};

package/scripts/rag-precision-guardrails.js

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+'use strict';
+
+const { listGateTemplates } = require('./gate-templates');
+
+const DOCUMENT_RAG_CATEGORY = 'Document RAG Safety';
+const PRECISION_TEMPLATE_IDS = new Set([
+  'require-rag-baseline-before-precision-tuning',
+  'require-two-stage-rag-verifier-for-structural-near-misses',
+  'checkpoint-rag-latency-precision-tradeoff',
+]);
+
+function normalizeBoolean(value) {
+  if (value === true) return true;
+  if (value === false || value === undefined || value === null) return false;
+  return /^(1|true|yes|on)$/i.test(String(value).trim());
+}
+
+function toNumber(value) {
+  if (value === undefined || value === null || value === '') return null;
+  const num = Number(value);
+  return Number.isFinite(num) ? num : null;
+}
+
+function normalizeOptions(options = {}) {
+  return {
+    ragTool: String(options['rag-tool'] || options.tool || 'agentic-rag').trim() || 'agentic-rag',
+    baselineRecall: toNumber(options['baseline-recall'] || options.recall),
+    newRecall: toNumber(options['new-recall']),
+    baselinePrecision: toNumber(options['baseline-precision'] || options.precision),
+    newPrecision: toNumber(options['new-precision']),
+    topK: toNumber(options['top-k'] || options.k),
+    thresholdChanged: normalizeBoolean(options['threshold-change'] || options['threshold-changed']),
+    embeddingFineTune: normalizeBoolean(options['embedding-finetune'] || options['embedding-fine-tune'] || options['fine-tune']),
+    structuralNearMisses: normalizeBoolean(options['structural-near-misses'] || options['near-misses']),
+    verifier: normalizeBoolean(options.verifier || options.reranker || options['second-stage']),
+    latencyMs: toNumber(options['latency-ms'] || options.latency),
+    latencyBudgetMs: toNumber(options['latency-budget-ms'] || options['latency-budget']),
+    agenticPipeline: normalizeBoolean(options.agentic || options['agentic-pipeline']),
+  };
+}
+
+function recallDropPercent(options) {
+  if (options.baselineRecall === null || options.newRecall === null || options.baselineRecall <= 0) return null;
+  return Number((((options.baselineRecall - options.newRecall) / options.baselineRecall) * 100).toFixed(2));
+}
+
+function templateApplicability(template, options) {
+  if (template.id === 'require-rag-baseline-before-precision-tuning') {
+    return options.thresholdChanged ||
+      options.embeddingFineTune ||
+      options.baselineRecall === null ||
+      options.newRecall === null ||
+      (recallDropPercent(options) !== null && recallDropPercent(options) > 5);
+  }
+  if (template.id === 'require-two-stage-rag-verifier-for-structural-near-misses') {
+    return options.structuralNearMisses || (options.agenticPipeline && !options.verifier);
+  }
+  if (template.id === 'checkpoint-rag-latency-precision-tradeoff') {
+    return options.verifier ||
+      (options.latencyMs !== null && options.latencyBudgetMs !== null && options.latencyMs > options.latencyBudgetMs);
+  }
+  return false;
+}
+
+function buildSignals(options) {
+  const drop = recallDropPercent(options);
+  return [
+    precisionTuningSignal(options, drop),
+    ragCascadeSignal(options),
+    verifierLatencySignal(options),
+  ].filter(Boolean);
+}
+
+function precisionTuningSignal(options, drop) {
+  if (!(options.thresholdChanged || options.embeddingFineTune || drop !== null)) return null;
+  return {
+    id: 'precision_tuning',
+    label: 'Precision tuning change',
+    values: [
+      options.thresholdChanged ? 'threshold changed' : null,
+      options.embeddingFineTune ? 'embedding fine-tune' : null,
+      drop !== null ? `recall drop ${drop}%` : null,
+    ].filter(Boolean),
+    risk: 'precision wins can hide broad retrieval recall regressions',
+  };
+}
+
+function ragCascadeSignal(options) {
+  if (!(options.structuralNearMisses || options.agenticPipeline)) return null;
+  return {
+    id: 'agentic_rag_cascade',
+    label: 'Agentic RAG cascade risk',
+    values: [
+      options.agenticPipeline ? 'agentic pipeline' : null,
+      options.structuralNearMisses ? 'structural near misses' : null,
+    ].filter(Boolean),
+    risk: 'wrong retrieval can trigger downstream tool calls, not just wrong answers',
+  };
+}
+
+function verifierLatencySignal(options) {
+  if (!(options.verifier || options.latencyMs !== null || options.latencyBudgetMs !== null)) return null;
+  return {
+    id: 'verifier_latency',
+    label: 'Verifier latency tradeoff',
+    values: [
+      options.verifier ? 'verifier enabled' : null,
+      options.latencyMs !== null ? `${options.latencyMs}ms observed` : null,
+      options.latencyBudgetMs !== null ? `${options.latencyBudgetMs}ms budget` : null,
+    ].filter(Boolean),
+    risk: 'second-stage verification needs a known latency budget',
+  };
+}
+
+function buildRagPrecisionGuardrailsPlan(rawOptions = {}, templatesPath) {
+  const options = normalizeOptions(rawOptions);
+  const templates = listGateTemplates(templatesPath)
+    .filter((template) => template.category === DOCUMENT_RAG_CATEGORY && PRECISION_TEMPLATE_IDS.has(template.id))
+    .map((template) => ({
+      ...template,
+      recommended: templateApplicability(template, options),
+    }));
+  const signals = buildSignals(options);
+  const recommendedTemplates = templates.filter((template) => template.recommended);
+
+  return {
+    name: 'thumbgate-rag-precision-guardrails',
+    status: recommendedTemplates.length > 0 ? 'actionable' : 'ready',
+    ragTool: options.ragTool,
+    metrics: {
+      topK: options.topK,
+      baselineRecall: options.baselineRecall,
+      newRecall: options.newRecall,
+      recallDropPercent: recallDropPercent(options),
+      baselinePrecision: options.baselinePrecision,
+      newPrecision: options.newPrecision,
+      latencyMs: options.latencyMs,
+      latencyBudgetMs: options.latencyBudgetMs,
+    },
+    summary: {
+      signalCount: signals.length,
+      templateCount: templates.length,
+      recommendedTemplateCount: recommendedTemplates.length,
+    },
+    signals,
+    templates,
+    nextActions: [
+      'Save baseline recall@k, precision@k, answer-with-evidence, and latency before tuning retrieval.',
+      'Block embedding or threshold changes when recall drops without an approved rollback plan.',
+      'Use a second-stage verifier or reranker for structural near misses such as negation and role reversal.',
+      'Attach verifier latency budgets before routing the retrieval output into autonomous agent actions.',
+    ],
+    exampleCommand: 'npx thumbgate rag-precision-guardrails --baseline-recall=0.86 --new-recall=0.72 --threshold-change --agentic --structural-near-misses --json',
+  };
+}
+
+function formatRagPrecisionGuardrailsPlan(report) {
+  const lines = [
+    '',
+    'ThumbGate RAG Precision Guardrails',
+    '-'.repeat(39),
+    `Status   : ${report.status}`,
+    `RAG tool : ${report.ragTool}`,
+    `Signals  : ${report.summary.signalCount}`,
+    `Templates: ${report.summary.recommendedTemplateCount}/${report.summary.templateCount} recommended`,
+  ];
+  if (report.metrics.recallDropPercent !== null) {
+    lines.push(`Recall drop: ${report.metrics.recallDropPercent}%`);
+  }
+
+  if (report.signals.length > 0) {
+    lines.push('', 'Detected retrieval risk signals:');
+    for (const signal of report.signals) {
+      lines.push(`  - ${signal.label}: ${signal.values.join(', ')}`);
+      lines.push(`    Risk: ${signal.risk}`);
+    }
+  }
+
+  lines.push('', 'Recommended templates:');
+  const recommended = report.templates.filter((template) => template.recommended);
+  if (recommended.length === 0) {
+    lines.push('  - No precision-risk signals were passed. Start with recall metrics, threshold changes, or verifier flags.');
+  } else {
+    for (const template of recommended) {
+      lines.push(`  - ${template.id} [${template.defaultAction}]`);
+      lines.push(`    ${template.roi}`);
+    }
+  }
+
+  lines.push('', 'Next actions:');
+  for (const action of report.nextActions) lines.push(`  - ${action}`);
+  lines.push('', `Example: ${report.exampleCommand}`, '');
+  return `${lines.join('\n')}\n`;
+}
+
+module.exports = {
+  buildRagPrecisionGuardrailsPlan,
+  formatRagPrecisionGuardrailsPlan,
+  normalizeOptions,
+  recallDropPercent,
+};

package/scripts/rate-limiter.js

@@ -36,7 +36,7 @@ const PAYWALL_MESSAGES = {
   prevention_rules: 'Free tier includes 1 prevention rule. Your agents need more protection — upgrade to Pro for unlimited rules.',
   recall: 'Recall is a Pro feature. Your past feedback is stored locally — upgrade to search and reuse it.',
   search_lessons: 'Lesson search is a Pro feature. Upgrade to find patterns in your agent\'s mistakes.',
-  default: 'This feature requires Pro. Start a 7-day free trial — no credit card required.',
+  default: 'This feature requires Pro. Start a 7-day trial — card required; no charge today.',
 };
 
 function isProTier(authContext) {

package/scripts/reasoning-efficiency-guardrails.js

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+'use strict';
+
+const { listGateTemplates } = require('./gate-templates');
+
+const CATEGORY = 'Reasoning Efficiency Safety';
+
+function normalizeBoolean(value) {
+  if (value === true) return true;
+  if (value === false || value === undefined || value === null) return false;
+  return /^(1|true|yes|on)$/i.test(String(value).trim());
+}
+
+function toNumber(value) {
+  if (value === undefined || value === null || value === '') return null;
+  const num = Number(value);
+  return Number.isFinite(num) ? num : null;
+}
+
+function normalizeOptions(options = {}) {
+  return {
+    workload: String(options.workload || options.name || 'reasoning').trim() || 'reasoning',
+    baselineTokens: toNumber(options['baseline-tokens']),
+    compressedTokens: toNumber(options['compressed-tokens']),
+    baselineAccuracy: toNumber(options['baseline-accuracy'] || options['baseline-pass1']),
+    compressedAccuracy: toNumber(options['compressed-accuracy'] || options['compressed-pass1']),
+    verifier: normalizeBoolean(options.verifier || options['verification-outcomes']),
+    lowConfidenceSteps: toNumber(options['low-confidence-steps']),
+    highConfidenceFailures: toNumber(options['high-confidence-failures']),
+    truncationFailures: normalizeBoolean(options['truncation-failures']),
+  };
+}
+
+function tokenReductionPercent(options) {
+  if (options.baselineTokens === null || options.compressedTokens === null || options.baselineTokens <= 0) return null;
+  return Number((((options.baselineTokens - options.compressedTokens) / options.baselineTokens) * 100).toFixed(2));
+}
+
+function accuracyDelta(options) {
+  if (options.baselineAccuracy === null || options.compressedAccuracy === null) return null;
+  return Number((options.compressedAccuracy - options.baselineAccuracy).toFixed(4));
+}
+
+function templateApplicability(template, options) {
+  if (template.id === 'require-verifier-before-reasoning-compression') {
+    return !options.verifier || accuracyDelta(options) === null || (accuracyDelta(options) !== null && accuracyDelta(options) < -0.01);
+  }
+  if (template.id === 'checkpoint-low-confidence-reasoning-steps') {
+    return options.lowConfidenceSteps !== null && options.lowConfidenceSteps > 0;
+  }
+  if (template.id === 'checkpoint-high-confidence-failed-rollout') {
+    return (options.highConfidenceFailures !== null && options.highConfidenceFailures > 0) || options.truncationFailures;
+  }
+  return false;
+}
+
+function buildSignals(options) {
+  const signals = [];
+  const tokenReduction = tokenReductionPercent(options);
+  const delta = accuracyDelta(options);
+  if (tokenReduction !== null || delta !== null || !options.verifier) {
+    signals.push({
+      id: 'reasoning_compression',
+      label: 'Reasoning compression rollout',
+      values: [
+        tokenReduction !== null ? `${tokenReduction}% token reduction` : null,
+        delta !== null ? `${delta} accuracy delta` : null,
+        options.verifier ? 'verifier present' : 'missing verifier',
+      ].filter(Boolean),
+      risk: 'shorter traces can reduce cost while destabilizing accuracy',
+    });
+  }
+  if (options.lowConfidenceSteps !== null && options.lowConfidenceSteps > 0) {
+    signals.push({
+      id: 'low_confidence_steps',
+      label: 'Low-confidence accepted steps',
+      values: [`${options.lowConfidenceSteps} step(s)`],
+      risk: 'successful rollouts can still contain brittle intermediate reasoning',
+    });
+  }
+  if ((options.highConfidenceFailures !== null && options.highConfidenceFailures > 0) || options.truncationFailures) {
+    signals.push({
+      id: 'failed_confident_rollouts',
+      label: 'High-confidence failed rollout',
+      values: [
+        options.highConfidenceFailures !== null ? `${options.highConfidenceFailures} failure(s)` : null,
+        options.truncationFailures ? 'truncation failure' : null,
+      ].filter(Boolean),
+      risk: 'failed rollouts may reflect verifier noise or truncation rather than bad reasoning',
+    });
+  }
+  return signals;
+}
+
+function buildReasoningEfficiencyGuardrailsPlan(rawOptions = {}, templatesPath) {
+  const options = normalizeOptions(rawOptions);
+  const templates = listGateTemplates(templatesPath)
+    .filter((template) => template.category === CATEGORY)
+    .map((template) => ({
+      ...template,
+      recommended: templateApplicability(template, options),
+    }));
+  const signals = buildSignals(options);
+  const recommendedTemplates = templates.filter((template) => template.recommended);
+
+  return {
+    name: 'thumbgate-reasoning-efficiency-guardrails',
+    status: recommendedTemplates.length > 0 ? 'actionable' : 'ready',
+    workload: options.workload,
+    metrics: {
+      baselineTokens: options.baselineTokens,
+      compressedTokens: options.compressedTokens,
+      tokenReductionPercent: tokenReductionPercent(options),
+      baselineAccuracy: options.baselineAccuracy,
+      compressedAccuracy: options.compressedAccuracy,
+      accuracyDelta: accuracyDelta(options),
+    },
+    summary: {
+      signalCount: signals.length,
+      templateCount: templates.length,
+      recommendedTemplateCount: recommendedTemplates.length,
+    },
+    signals,
+    templates,
+    nextActions: [
+      'Keep a verifier and pass@1 baseline before compressing reasoning traces.',
+      'Inspect low-confidence steps even when the final rollout is correct.',
+      'Inspect high-confidence failed rollouts for truncation or verifier noise before penalizing the trace.',
+      'Route cheaper compressed reasoning only after accuracy and efficiency both clear the gate.',
+    ],
+    exampleCommand: 'npx thumbgate reasoning-efficiency-guardrails --baseline-tokens=1200 --compressed-tokens=980 --baseline-accuracy=0.84 --compressed-accuracy=0.85 --verifier --json',
+  };
+}
+
+function formatReasoningEfficiencyGuardrailsPlan(report) {
+  const lines = [
+    '',
+    'ThumbGate Reasoning Efficiency Guardrails',
+    '-'.repeat(43),
+    `Status  : ${report.status}`,
+    `Workload: ${report.workload}`,
+    `Signals : ${report.summary.signalCount}`,
+    `Templates: ${report.summary.recommendedTemplateCount}/${report.summary.templateCount} recommended`,
+  ];
+  if (report.metrics.tokenReductionPercent !== null) lines.push(`Token reduction: ${report.metrics.tokenReductionPercent}%`);
+  if (report.metrics.accuracyDelta !== null) lines.push(`Accuracy delta : ${report.metrics.accuracyDelta}`);
+
+  if (report.signals.length > 0) {
+    lines.push('', 'Detected reasoning-efficiency signals:');
+    for (const signal of report.signals) {
+      lines.push(`  - ${signal.label}: ${signal.values.join(', ')}`);
+      lines.push(`    Risk: ${signal.risk}`);
+    }
+  }
+
+  lines.push('', 'Recommended templates:');
+  const recommended = report.templates.filter((template) => template.recommended);
+  if (recommended.length === 0) lines.push('  - No reasoning-efficiency risks were passed.');
+  for (const template of recommended) {
+    lines.push(`  - ${template.id} [${template.defaultAction}]`);
+    lines.push(`    ${template.roi}`);
+  }
+
+  lines.push('', 'Next actions:');
+  for (const action of report.nextActions) lines.push(`  - ${action}`);
+  lines.push('', `Example: ${report.exampleCommand}`, '');
+  return `${lines.join('\n')}\n`;
+}
+
+module.exports = {
+  buildReasoningEfficiencyGuardrailsPlan,
+  formatReasoningEfficiencyGuardrailsPlan,
+  normalizeOptions,
+  tokenReductionPercent,
+  accuracyDelta,
+};