thumbgate 1.16.13 → 1.16.19

package/scripts/proactive-agent-eval-guardrails.js

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SOURCE = Object.freeze({
+  postUrl: 'https://x.com/dair_ai/status/2048083882327916688',
+  paperUrl: 'https://arxiv.org/abs/2604.00842',
+  title: 'Proactive Agent Research Environment: Simulating Active Users to Evaluate Proactive Assistants',
+  arxivId: '2604.00842',
+  submitted: '2026-04-01',
+  postCreatedAt: '2026-04-25T16:57:03.000Z',
+});
+
+function parseNumber(value, fallback = 0) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function parseBoolean(value, fallback = false) {
+  if (value === undefined || value === null || value === '') return fallback;
+  if (typeof value === 'boolean') return value;
+  return /^(1|true|yes|on)$/i.test(String(value).trim());
+}
+
+function splitList(value) {
+  if (Array.isArray(value)) return value.map(String).map((item) => item.trim()).filter(Boolean);
+  return String(value || '').split(',').map((item) => item.trim()).filter(Boolean);
+}
+
+function normalizeOptions(raw = {}) {
+  return {
+    workflow: String(raw.workflow || raw.name || 'proactive agent workflow').trim() || 'proactive agent workflow',
+    apps: splitList(raw.apps || raw.applications),
+    states: splitList(raw.states || raw['app-states']),
+    stateCount: parseNumber(raw['state-count'] || raw.stateCount || splitList(raw.states).length, splitList(raw.states).length),
+    actionCount: parseNumber(raw['action-count'] || raw.actionCount, 0),
+    taskCount: parseNumber(raw['task-count'] || raw.taskCount, 0),
+    hasStateMachine: parseBoolean(raw['state-machine'] || raw.hasStateMachine, false),
+    hasActiveUserSimulation: parseBoolean(raw['active-user-simulation'] || raw.hasActiveUserSimulation, false),
+    hasGoalInferenceEvals: parseBoolean(raw['goal-inference-evals'] || raw.hasGoalInferenceEvals, false),
+    hasInterventionTimingEvals: parseBoolean(raw['intervention-timing-evals'] || raw.hasInterventionTimingEvals, false),
+    hasMultiAppEvals: parseBoolean(raw['multi-app-evals'] || raw.hasMultiAppEvals, false),
+    flatToolApiOnly: parseBoolean(raw['flat-tool-api-only'] || raw.flatToolApiOnly, false),
+    proactiveWrites: parseBoolean(raw['proactive-writes'] || raw.proactiveWrites, false),
+    userVisibleActions: parseBoolean(raw['user-visible-actions'] || raw.userVisibleActions, false),
+  };
+}
+
+function buildSignals(options) {
+  const signals = [];
+  if (!options.hasStateMachine || options.flatToolApiOnly) {
+    signals.push({
+      id: 'flat_tool_api_gap',
+      severity: 'high',
+      message: 'Flat tool APIs miss stateful navigation and state-dependent action spaces.',
+      gate: 'Require finite-state app model before proactive execution.',
+    });
+  }
+  if (!options.hasActiveUserSimulation) {
+    signals.push({
+      id: 'missing_active_user_simulation',
+      severity: 'high',
+      message: 'Proactive agents need simulated user progress before timing can be evaluated.',
+      gate: 'Run active user simulation before enabling anticipatory actions.',
+    });
+  }
+  if (!options.hasGoalInferenceEvals) {
+    signals.push({
+      id: 'missing_goal_inference_eval',
+      severity: 'medium',
+      message: 'The agent may intervene without evidence that it inferred the user goal correctly.',
+      gate: 'Grade goal inference before intervention approval.',
+    });
+  }
+  if (!options.hasInterventionTimingEvals) {
+    signals.push({
+      id: 'missing_intervention_timing_eval',
+      severity: 'high',
+      message: 'A helpful action at the wrong time becomes interruption or damage.',
+      gate: 'Require too-early, on-time, and too-late timing eval cases.',
+    });
+  }
+  if ((options.apps.length > 1 || options.hasMultiAppEvals === false) && options.proactiveWrites) {
+    signals.push({
+      id: 'multi_app_write_risk',
+      severity: 'critical',
+      message: 'Multi-app proactive writes can compound state mistakes across tools.',
+      gate: 'Block multi-app proactive writes until orchestration evals and rollback evidence exist.',
+    });
+  }
+  if (options.userVisibleActions && !options.hasInterventionTimingEvals) {
+    signals.push({
+      id: 'user_visible_interruption_risk',
+      severity: 'high',
+      message: 'User-visible interventions need timing proof before notification, scheduling, or communication actions.',
+      gate: 'Require intervention timing proof before user-visible actions.',
+    });
+  }
+  return signals;
+}
+
+function buildMetrics(options) {
+  return [
+    { id: 'goal_inference_accuracy', target: '>= 0.85', required: true },
+    { id: 'intervention_timing_f1', target: '>= 0.80', required: true },
+    { id: 'false_intervention_rate', target: '<= 0.05', required: true },
+    { id: 'state_transition_validity', target: '>= 0.98', required: true },
+    { id: 'multi_app_orchestration_success', target: options.apps.length > 1 ? '>= 0.85' : 'optional', required: options.apps.length > 1 },
+  ];
+}
+
+function buildProactiveAgentEvalGuardrailsPlan(rawOptions = {}) {
+  const options = normalizeOptions(rawOptions);
+  const signals = buildSignals(options);
+  const critical = signals.filter((signal) => signal.severity === 'critical').length;
+  const high = signals.filter((signal) => signal.severity === 'high').length;
+
+  return {
+    name: 'thumbgate-proactive-agent-eval-guardrails',
+    source: SOURCE,
+    workflow: options.workflow,
+    status: critical > 0 ? 'blocked' : high > 0 ? 'needs_eval' : 'ready',
+    summary: {
+      signalCount: signals.length,
+      critical,
+      high,
+      apps: options.apps,
+      stateCount: options.stateCount,
+      actionCount: options.actionCount,
+      taskCount: options.taskCount,
+    },
+    pareMapping: {
+      appModel: options.hasStateMachine ? 'finite_state_machine_present' : 'finite_state_machine_required',
+      userSimulation: options.hasActiveUserSimulation ? 'active_user_simulation_present' : 'active_user_simulation_required',
+      actionSpace: 'state-dependent action spaces should be explicit per app state',
+      evaluationAxes: [
+        'context observation',
+        'goal inference',
+        'intervention timing',
+        'multi-app orchestration',
+      ],
+    },
+    signals,
+    metrics: buildMetrics(options),
+    gates: signals.map((signal) => ({
+      id: signal.id,
+      action: signal.severity === 'critical' ? 'block' : 'warn',
+      message: signal.gate,
+    })),
+    nextActions: [
+      'Model each app as states, allowed actions, and valid transitions before judging proactive behavior.',
+      'Add active user simulation cases where the user keeps navigating while the agent observes.',
+      'Evaluate goal inference separately from intervention timing so a correct goal at the wrong time is still caught.',
+      'Block proactive writes across multiple apps until orchestration success and rollback evidence are measured.',
+      'Attach the eval report to any claim that a proactive agent is production-ready.',
+    ],
+    marketingAngle: {
+      headline: 'Proactive agents need stateful eval gates.',
+      subhead: 'PARE shows why flat tool-call benchmarks miss real app behavior. ThumbGate turns those stateful eval failures into pre-action gates before a proactive assistant interrupts users or writes across apps.',
+      replyDraft: 'This is the missing eval shape for proactive agents. Flat tool calls cannot tell whether the agent acted at the right state or the right time. ThumbGate can use this pattern as the enforcement layer: stateful eval failure -> pre-action gate before the next proactive write.',
+    },
+  };
+}
+
+function formatProactiveAgentEvalGuardrailsPlan(report) {
+  const lines = [
+    '',
+    'ThumbGate Proactive Agent Eval Guardrails',
+    '-'.repeat(43),
+    `Workflow : ${report.workflow}`,
+    `Status   : ${report.status}`,
+    `Source   : ${report.source.paperUrl}`,
+    `Signals  : ${report.summary.signalCount}`,
+  ];
+  if (report.signals.length > 0) {
+    lines.push('', 'Signals:');
+    for (const signal of report.signals) {
+      lines.push(`  - [${signal.severity}] ${signal.id}: ${signal.message}`);
+      lines.push(`    Gate: ${signal.gate}`);
+    }
+  }
+  lines.push('', 'Required metrics:');
+  for (const metric of report.metrics) {
+    lines.push(`  - ${metric.id}: ${metric.target}${metric.required ? ' (required)' : ''}`);
+  }
+  lines.push('', 'Next actions:');
+  for (const action of report.nextActions) lines.push(`  - ${action}`);
+  lines.push('', `Reply draft: ${report.marketingAngle.replyDraft}`, '');
+  return `${lines.join('\n')}\n`;
+}
+
+function writeProactiveAgentEvalPromoPack(outputDir = path.join(__dirname, '..', 'docs', 'marketing')) {
+  const report = buildProactiveAgentEvalGuardrailsPlan({
+    workflow: 'proactive assistant shipping checklist',
+    apps: ['calendar', 'email', 'tasks'],
+    'state-count': 12,
+    'action-count': 24,
+    'task-count': 143,
+    'flat-tool-api-only': true,
+    'proactive-writes': true,
+    'user-visible-actions': true,
+  });
+  fs.mkdirSync(outputDir, { recursive: true });
+  const jsonPath = path.join(outputDir, 'dair-pare-proactive-agent-pack.json');
+  const markdownPath = path.join(outputDir, 'dair-pare-proactive-agent-pack.md');
+  fs.writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`);
+  fs.writeFileSync(markdownPath, formatProactiveAgentEvalGuardrailsPlan(report));
+  return { report, jsonPath, markdownPath };
+}
+
+module.exports = {
+  SOURCE,
+  buildMetrics,
+  buildProactiveAgentEvalGuardrailsPlan,
+  buildSignals,
+  formatProactiveAgentEvalGuardrailsPlan,
+  normalizeOptions,
+  writeProactiveAgentEvalPromoPack,
+};
+
+function isCliInvocation(argv = process.argv) {
+  return Boolean(argv[1] && path.resolve(argv[1]) === __filename);
+}
+
+if (isCliInvocation()) {
+  const { jsonPath, markdownPath } = writeProactiveAgentEvalPromoPack();
+  console.log(JSON.stringify({ jsonPath, markdownPath }, null, 2));
+}

package/scripts/profile-router.js

@@ -125,14 +125,15 @@ function findMostRestrictiveProfile(toolName) {
   if (!policy || !policy.profiles) return null;
   const profileRank = new Map(PROFILE_RESTRICTIVENESS_ORDER.map((name, index) => [name, index]));
 
-  // Sort profiles by number of tools (most restrictive first)
+  // Sort by named least-privilege order first. Some specialized profiles
+  // expose fewer tools than locked but carry stronger side-effect authority.
   const candidates = Object.entries(policy.profiles)
     .filter(([, tools]) => tools.includes(toolName))
     .sort((a, b) => {
-      const sizeDelta = a[1].length - b[1].length;
-      if (sizeDelta !== 0) return sizeDelta;
-      return (profileRank.get(a[0]) ?? Number.MAX_SAFE_INTEGER)
+      const rankDelta = (profileRank.get(a[0]) ?? Number.MAX_SAFE_INTEGER)
         - (profileRank.get(b[0]) ?? Number.MAX_SAFE_INTEGER);
+      if (rankDelta !== 0) return rankDelta;
+      return a[1].length - b[1].length;
     });
 
   if (candidates.length === 0) return null;

package/scripts/prompting-operating-system.js

@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Prompting Operating System
+ *
+ * Converts modern "AI power user" prompting guidance into ThumbGate pre-action
+ * planning: choose the run mode, gather the right context, set reasoning
+ * budget, route tools, and attach trust checks before an agent acts.
+ */
+
+const path = require('node:path');
+
+const MODE_RULES = [
+  {
+    mode: 'deep-research',
+    pattern: /\b(research|report|compare|market|latest|sources?|citations?|well-researched)\b/i,
+    reasoningBudget: 'deep',
+    tools: ['web-search', 'source-capture', 'citation-check'],
+    contextRequirements: ['question', 'source scope', 'recency requirement'],
+  },
+  {
+    mode: 'decision-support',
+    pattern: /\b(decide|choose|buy|job|career|study|hire|pricing|strategy|important decision)\b/i,
+    reasoningBudget: 'minutes',
+    tools: ['tradeoff-table', 'risk-check', 'evidence-check'],
+    contextRequirements: ['decision criteria', 'constraints', 'stakes', 'alternatives'],
+  },
+  {
+    mode: 'build',
+    pattern: /\b(build|implement|code|website|game|app|fix|ship)\b/i,
+    reasoningBudget: 'standard',
+    tools: ['repo-inspection', 'code-edit', 'tests'],
+    contextRequirements: ['target files', 'acceptance criteria', 'verification command'],
+  },
+  {
+    mode: 'data-analysis',
+    pattern: /\b(analyze data|spreadsheet|csv|xlsx|chart|visuali[sz]e|dataset)\b/i,
+    reasoningBudget: 'standard',
+    tools: ['data-parser', 'charting', 'sanity-check'],
+    contextRequirements: ['data file', 'analysis question', 'metric definitions'],
+  },
+  {
+    mode: 'visual-generation',
+    pattern: /\b(image|photo|diagram|visual|mockup|poster|thumbnail)\b/i,
+    reasoningBudget: 'standard',
+    tools: ['image-generation', 'visual-qa'],
+    contextRequirements: ['visual goal', 'style direction', 'dimensions'],
+  },
+];
+
+const DEFAULT_MODE = {
+  mode: 'quick-answer',
+  reasoningBudget: 'fast',
+  tools: ['direct-answer'],
+  contextRequirements: ['question'],
+};
+
+function planPromptingRun(input = {}) {
+  const request = String(input.request || input.prompt || '').trim();
+  const attachments = Array.isArray(input.attachments) ? input.attachments : [];
+  const rule = MODE_RULES.find((candidate) => candidate.pattern.test(request)) || DEFAULT_MODE;
+  const attachmentContext = inferAttachmentContext(attachments);
+  const contextRequirements = Array.from(new Set([
+    ...rule.contextRequirements,
+    ...attachmentContext.requirements,
+  ]));
+  const providedContext = inferProvidedContext(request, attachments);
+  const missingContext = contextRequirements.filter((item) => !providedContext.includes(item));
+  const trustChecks = buildTrustChecks(rule.mode, input);
+
+  return {
+    mode: rule.mode,
+    reasoningBudget: rule.reasoningBudget,
+    tools: Array.from(new Set([...rule.tools, ...attachmentContext.tools])),
+    contextRequirements,
+    providedContext,
+    missingContext,
+    trustChecks,
+    gates: buildPromptGates({ mode: rule.mode, missingContext, trustChecks, input }),
+    promptTemplate: buildPromptTemplate({ mode: rule.mode, request, contextRequirements, trustChecks }),
+  };
+}
+
+function evaluatePromptReadiness(plan = {}) {
+  const hardBlocks = (plan.gates || []).filter((gate) => gate.severity === 'block');
+  const warnings = (plan.gates || []).filter((gate) => gate.severity === 'warn');
+  return {
+    allowed: hardBlocks.length === 0,
+    score: Math.max(0, 100 - (hardBlocks.length * 35) - (warnings.length * 10) - ((plan.missingContext || []).length * 8)),
+    hardBlocks,
+    warnings,
+    recommendation: promptReadinessRecommendation(hardBlocks, warnings),
+  };
+}
+
+function promptReadinessRecommendation(hardBlocks, warnings) {
+  if (hardBlocks.length) return 'Collect missing high-stakes context before running the agent.';
+  if (warnings.length) return 'Proceed with explicit uncertainty and evidence capture.';
+  return 'Prompt is ready for execution.';
+}
+
+function inferAttachmentContext(attachments = []) {
+  const requirements = [];
+  const tools = [];
+  for (const attachment of attachments) {
+    const name = String(attachment.name || attachment.path || attachment.type || '').toLowerCase();
+    if (/\.(png|jpg|jpeg|webp|gif)$|image/.test(name)) {
+      requirements.push('image context');
+      tools.push('vision');
+    }
+    if (/\.(csv|tsv|xlsx|xls)$|spreadsheet|dataset/.test(name)) {
+      requirements.push('data file');
+      tools.push('data-parser');
+    }
+    if (/\.(pdf|docx|md|txt)$|document/.test(name)) {
+      requirements.push('document context');
+      tools.push('document-parser');
+    }
+  }
+  return { requirements, tools };
+}
+
+function inferProvidedContext(request, attachments = []) {
+  const provided = new Set();
+  if (request) provided.add('question');
+  if (/\b(criteria|must|need|constraint|budget|deadline)\b/i.test(request)) provided.add('constraints');
+  if (/\b(option|alternative|versus|vs\.?|compare)\b/i.test(request)) provided.add('alternatives');
+  if (/\b(source|cite|link|latest|2026|today|yesterday)\b/i.test(request)) {
+    provided.add('source scope');
+    provided.add('recency requirement');
+  }
+  if (/\b(test|verify|acceptance|done when)\b/i.test(request)) provided.add('verification command');
+  if (/\b(file|path|component|script|module)\b/i.test(request) || hasPathLikeTarget(request)) {
+    provided.add('target files');
+  }
+  if (/\b(metric|kpi|revenue|conversion|score)\b/i.test(request)) provided.add('metric definitions');
+  if (attachments.length > 0) {
+    provided.add('document context');
+    provided.add('image context');
+    provided.add('data file');
+  }
+  return Array.from(provided);
+}
+
+function hasPathLikeTarget(request = '') {
+  return String(request)
+    .split(/\s+/)
+    .map(stripPathPunctuation)
+    .some((part) => part.includes('/') && part.includes('.') && isSafePathToken(part));
+}
+
+function stripPathPunctuation(part) {
+  let text = String(part);
+  while (text && '("\''.includes(text[0])) text = text.slice(1);
+  while (text && '),.\'"'.includes(text.at(-1))) text = text.slice(0, -1);
+  return text;
+}
+
+function isSafePathToken(part) {
+  return String(part).split('').every((char) => {
+    const code = char.charCodeAt(0);
+    return char === '/' || char === '.' || char === '-' || char === '_' ||
+      (code >= 48 && code <= 57) ||
+      (code >= 65 && code <= 90) ||
+      (code >= 97 && code <= 122);
+  });
+}
+
+function buildTrustChecks(mode, input = {}) {
+  const checks = [
+    'state assumptions explicitly',
+    'capture evidence before completion claim',
+    'mark uncertainty when source quality is weak',
+  ];
+  if (mode === 'deep-research') checks.push('cite primary or authoritative sources', 'compare publication dates');
+  if (mode === 'decision-support') checks.push('show tradeoffs and downside risk', 'separate reversible from irreversible choices');
+  if (mode === 'build') checks.push('inspect code before editing', 'run focused tests');
+  if (mode === 'data-analysis') checks.push('validate row counts and missing values', 'avoid unsupported causal claims');
+  if (input.highStakes) checks.push('escalate high-stakes claims for verification');
+  return checks;
+}
+
+function buildPromptGates({ mode, missingContext, trustChecks, input }) {
+  const gates = [];
+  const highStakes = Boolean(input.highStakes || ['decision-support'].includes(mode));
+  if (highStakes && missingContext.length > 0) {
+    gates.push({
+      id: 'missing-high-stakes-context',
+      severity: 'block',
+      reason: `Missing context for high-stakes ${mode}: ${missingContext.join(', ')}`,
+    });
+  }
+  if (missingContext.length > 0) {
+    gates.push({
+      id: 'missing-context-warning',
+      severity: 'warn',
+      reason: `Missing context may reduce answer quality: ${missingContext.join(', ')}`,
+    });
+  }
+  if (trustChecks.includes('capture evidence before completion claim')) {
+    gates.push({
+      id: 'evidence-before-claim',
+      severity: 'warn',
+      reason: 'Do not claim completion without evidence.',
+    });
+  }
+  return gates;
+}
+
+function buildPromptTemplate({ mode, request, contextRequirements, trustChecks }) {
+  return [
+    `Mode: ${mode}`,
+    `Task: ${request || '<task>'}`,
+    `Context to provide: ${contextRequirements.join(', ')}`,
+    `Trust checks: ${trustChecks.join('; ')}`,
+    'Output: answer with evidence, assumptions, and next action.',
+  ].join('\n');
+}
+
+function formatPromptingPlan(plan = {}) {
+  return [
+    '# Prompting Operating Plan',
+    '',
+    `Mode: ${plan.mode}`,
+    `Reasoning budget: ${plan.reasoningBudget}`,
+    `Tools: ${(plan.tools || []).join(', ')}`,
+    `Missing context: ${(plan.missingContext || []).join(', ') || 'none'}`,
+    '',
+    '## Gates',
+    '',
+    ...(plan.gates || []).map((gate) => `- ${gate.severity}: ${gate.id} — ${gate.reason}`),
+    '',
+    '## Prompt Template',
+    '',
+    '```text',
+    plan.promptTemplate || '',
+    '```',
+    '',
+  ].join('\n');
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = { command: argv[0] || 'plan', request: '' };
+  for (const arg of argv.slice(1)) {
+    if (arg.startsWith('--request=')) args.request = arg.slice('--request='.length);
+    if (arg === '--high-stakes') args.highStakes = true;
+  }
+  return args;
+}
+
+function isCliInvocation(argv = process.argv) {
+  return Boolean(argv[1] && path.resolve(argv[1]) === __filename);
+}
+
+if (isCliInvocation()) {
+  const args = parseArgs();
+  const plan = planPromptingRun(args);
+  if (args.command === 'json') {
+    console.log(JSON.stringify({ plan, readiness: evaluatePromptReadiness(plan) }, null, 2));
+  } else if (args.command === 'plan') {
+    console.log(formatPromptingPlan(plan));
+  } else {
+    console.error(`Unknown command: ${args.command}. Use: plan, json`);
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  evaluatePromptReadiness,
+  formatPromptingPlan,
+  planPromptingRun,
+};