thumbgate 1.29.1 → 1.29.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/.claude/commands/dashboard.md +11 -1
  2. package/.claude/commands/thumbgate-dashboard.md +23 -8
  3. package/.claude-plugin/plugin.json +1 -1
  4. package/.well-known/mcp/server-card.json +1 -1
  5. package/README.md +61 -1
  6. package/adapters/claude/.mcp.json +2 -2
  7. package/adapters/forge/forge.yaml +3 -3
  8. package/adapters/mcp/server-stdio.js +88 -2
  9. package/adapters/opencode/opencode.json +1 -1
  10. package/commands/dashboard.md +11 -1
  11. package/commands/thumbgate-dashboard.md +23 -8
  12. package/config/agent-outcome-monitor-thresholds.json +63 -0
  13. package/config/evals/agent-outcomes-baseline.json +17 -0
  14. package/config/evals/agent-outcomes-golden.json +412 -0
  15. package/config/evals/prompt-eval-baseline.json +23 -0
  16. package/config/schemas/task-outcome-receipt.schema.json +296 -0
  17. package/openapi/openapi.yaml +235 -0
  18. package/package.json +19 -6
  19. package/public/index.html +4 -2
  20. package/public/numbers.html +2 -2
  21. package/scripts/agent-outcome-eval.js +130 -0
  22. package/scripts/agent-outcome-monitor.js +261 -0
  23. package/scripts/agent-reasoning-traces.js +8 -9
  24. package/scripts/async-job-runner.js +107 -13
  25. package/scripts/durability/step.js +121 -12
  26. package/scripts/gates-engine.js +431 -18
  27. package/scripts/human-escalation.js +265 -0
  28. package/scripts/hybrid-feedback-context.js +93 -50
  29. package/scripts/judge-reward-function.js +30 -18
  30. package/scripts/prompt-eval.js +81 -4
  31. package/scripts/schedule-manager.js +249 -0
  32. package/scripts/task-outcomes.js +425 -0
  33. package/scripts/tool-contract-validator.js +287 -59
  34. package/scripts/tool-registry.js +143 -0
  35. package/src/api/server.js +127 -5
package/src/api/server.js CHANGED
@@ -166,6 +166,19 @@ const {
166
166
  const {
167
167
  evaluateWorkflowSentinel,
168
168
  } = require('../../scripts/workflow-sentinel');
169
+ const {
170
+ calculateTaskOutcomeMetrics,
171
+ getTaskOutcome,
172
+ readTaskOutcomes,
173
+ recordTaskOutcome,
174
+ } = require('../../scripts/task-outcomes');
175
+ const { monitorTaskOutcomes } = require('../../scripts/agent-outcome-monitor');
176
+ const {
177
+ calculateEscalationMetrics,
178
+ decideEscalation,
179
+ listEscalations,
180
+ requestEscalation,
181
+ } = require('../../scripts/human-escalation');
169
182
  const {
170
183
  normalizeProviderAction,
171
184
  } = require('../../scripts/provider-action-normalizer');
@@ -3875,6 +3888,8 @@ function buildHostedRuntimePresence(hostedConfig, { expectedApiKey, expectedOper
3875
3888
  THUMBGATE_FEEDBACK_DIR: Boolean(process.env.THUMBGATE_FEEDBACK_DIR),
3876
3889
  THUMBGATE_OPERATOR_KEY: Boolean(expectedOperatorKey),
3877
3890
  THUMBGATE_API_KEY: Boolean(expectedApiKey),
3891
+ THUMBGATE_HUMAN_REVIEWER_KEY: Boolean(process.env.THUMBGATE_HUMAN_REVIEWER_KEY),
3892
+ THUMBGATE_HUMAN_REVIEWER_ID: Boolean(process.env.THUMBGATE_HUMAN_REVIEWER_ID),
3878
3893
  THUMBGATE_PUBLIC_APP_ORIGIN: Boolean(hostedConfig.appOrigin),
3879
3894
  THUMBGATE_BILLING_API_BASE_URL: Boolean(hostedConfig.billingApiBaseUrl),
3880
3895
  THUMBGATE_GA_MEASUREMENT_ID: Boolean(hostedConfig.gaMeasurementId),
@@ -4890,6 +4905,15 @@ function getExpectedOperatorKey() {
4890
4905
  return key || null;
4891
4906
  }
4892
4907
 
4908
+ function getHumanReviewerConfig(expectedApiKey, expectedOperatorKey) {
4909
+ const key = String(process.env.THUMBGATE_HUMAN_REVIEWER_KEY || '').trim();
4910
+ const id = String(process.env.THUMBGATE_HUMAN_REVIEWER_ID || '').trim();
4911
+ const independent = Boolean(key)
4912
+ && !safeKeyEqual(key, expectedApiKey)
4913
+ && !safeKeyEqual(key, expectedOperatorKey);
4914
+ return { key: key || null, id: id || null, independent };
4915
+ }
4916
+
4893
4917
  function isAuthorized(req, expected) {
4894
4918
  if (!expected) return true;
4895
4919
  const token = extractApiKey(req);
@@ -4957,6 +4981,21 @@ function safeKeyEqual(a, b) {
4957
4981
  return require('crypto').timingSafeEqual(x, y);
4958
4982
  }
4959
4983
 
4984
+ function authenticateHumanReviewer(req, reviewerConfig) {
4985
+ if (!reviewerConfig.key || !reviewerConfig.id || !reviewerConfig.independent) {
4986
+ throw createHttpError(503, 'Human escalation decisions are unavailable until an independent reviewer key and identity are configured');
4987
+ }
4988
+ const header = req.headers['x-thumbgate-human-reviewer-key'];
4989
+ const presented = Array.isArray(header) ? header[0] : header;
4990
+ if (!safeKeyEqual(presented, reviewerConfig.key)) {
4991
+ throw createHttpError(403, 'A valid human reviewer credential is required to decide an escalation');
4992
+ }
4993
+ return {
4994
+ id: reviewerConfig.id,
4995
+ kind: 'human',
4996
+ };
4997
+ }
4998
+
4960
4999
  function resolveKeyRole(key) {
4961
5000
  const k = String(key || '').trim();
4962
5001
  const adminKey = String(process.env.THUMBGATE_API_KEY || '').trim();
@@ -5143,6 +5182,7 @@ function resolveDocumentImportFilePath(inputPath, options = {}) {
5143
5182
  function createApiServer() {
5144
5183
  const expectedApiKey = getExpectedApiKey();
5145
5184
  const expectedOperatorKey = getExpectedOperatorKey();
5185
+ const humanReviewerConfig = getHumanReviewerConfig(expectedApiKey, expectedOperatorKey);
5146
5186
 
5147
5187
  const eventBus = new EventEmitter();
5148
5188
  eventBus.setMaxListeners(200);
@@ -8442,14 +8482,19 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
8442
8482
  return;
8443
8483
  }
8444
8484
 
8445
- // Operator key is allowed to bypass the general admin gate for its dedicated endpoint
8485
+ // Operator key is allowed to bypass the general admin gate for dedicated
8486
+ // read-only operational endpoints.
8446
8487
  const _reqToken = extractApiKey(req);
8447
- const isOperatorBillingRequest = Boolean(expectedOperatorKey)
8448
- && _reqToken === expectedOperatorKey
8488
+ const isOperatorReadRequest = Boolean(expectedOperatorKey)
8489
+ && safeKeyEqual(_reqToken, expectedOperatorKey)
8449
8490
  && req.method === 'GET'
8450
- && ['/v1/billing/summary', '/v1/intake/workflow-sprint/queue'].includes(pathname);
8491
+ && [
8492
+ '/v1/billing/summary',
8493
+ '/v1/intake/workflow-sprint/queue',
8494
+ '/v1/task-outcomes/monitor',
8495
+ ].includes(pathname);
8451
8496
 
8452
- if (!isOperatorBillingRequest && !isAuthorized(req, expectedApiKey)) {
8497
+ if (!isOperatorReadRequest && !isAuthorized(req, expectedApiKey)) {
8453
8498
  sendProblem(res, {
8454
8499
  type: PROBLEM_TYPES.UNAUTHORIZED,
8455
8500
  title: 'Unauthorized',
@@ -9037,6 +9082,83 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
9037
9082
  return;
9038
9083
  }
9039
9084
 
9085
+ if (req.method === 'POST' && pathname === '/v1/task-outcomes') {
9086
+ const body = await parseJsonBody(req);
9087
+ try {
9088
+ const result = recordTaskOutcome(body, { feedbackDir: requestFeedbackDir });
9089
+ sendJson(res, result.recorded ? 201 : 200, result);
9090
+ } catch (error) {
9091
+ throw createHttpError(400, error.message);
9092
+ }
9093
+ return;
9094
+ }
9095
+
9096
+ if (req.method === 'GET' && pathname === '/v1/task-outcomes') {
9097
+ const taskId = parsed.searchParams.get('taskId');
9098
+ const limit = Math.max(1, Math.min(100, Number(parsed.searchParams.get('limit') || 20)));
9099
+ if (taskId) {
9100
+ const outcome = getTaskOutcome(taskId, { feedbackDir: requestFeedbackDir });
9101
+ if (!outcome) throw createHttpError(404, `Task outcome not found: ${taskId}`);
9102
+ sendJson(res, 200, outcome);
9103
+ } else {
9104
+ sendJson(res, 200, {
9105
+ outcomes: readTaskOutcomes({ feedbackDir: requestFeedbackDir }).slice(-limit),
9106
+ });
9107
+ }
9108
+ return;
9109
+ }
9110
+
9111
+ if (req.method === 'GET' && pathname === '/v1/task-outcomes/metrics') {
9112
+ const outcomes = readTaskOutcomes({ feedbackDir: requestFeedbackDir });
9113
+ sendJson(res, 200, calculateTaskOutcomeMetrics(outcomes));
9114
+ return;
9115
+ }
9116
+
9117
+ if (req.method === 'GET' && pathname === '/v1/task-outcomes/monitor') {
9118
+ const outcomes = readTaskOutcomes({ feedbackDir: requestFeedbackDir });
9119
+ sendJson(res, 200, monitorTaskOutcomes(outcomes));
9120
+ return;
9121
+ }
9122
+
9123
+ if (req.method === 'POST' && pathname === '/v1/escalations') {
9124
+ const body = await parseJsonBody(req);
9125
+ try {
9126
+ const result = requestEscalation(body, { feedbackDir: requestFeedbackDir });
9127
+ sendJson(res, result.recorded ? 201 : 200, result);
9128
+ } catch (error) {
9129
+ throw createHttpError(400, error.message);
9130
+ }
9131
+ return;
9132
+ }
9133
+
9134
+ if (req.method === 'GET' && pathname === '/v1/escalations') {
9135
+ const status = parsed.searchParams.get('status') || undefined;
9136
+ const escalations = listEscalations({ feedbackDir: requestFeedbackDir, status });
9137
+ sendJson(res, 200, {
9138
+ escalations,
9139
+ metrics: calculateEscalationMetrics(escalations),
9140
+ });
9141
+ return;
9142
+ }
9143
+
9144
+ if (req.method === 'POST' && /^\/v1\/escalations\/[^/]+\/decision$/.test(pathname)) {
9145
+ const escalationId = decodeURIComponent(pathname.split('/')[3]);
9146
+ const authenticatedActor = authenticateHumanReviewer(req, humanReviewerConfig);
9147
+ const body = await parseJsonBody(req);
9148
+ try {
9149
+ sendJson(res, 200, decideEscalation({
9150
+ ...body,
9151
+ escalationId,
9152
+ }, {
9153
+ authenticatedActor,
9154
+ feedbackDir: requestFeedbackDir,
9155
+ }));
9156
+ } catch (error) {
9157
+ throw createHttpError(400, error.message);
9158
+ }
9159
+ return;
9160
+ }
9161
+
9040
9162
  if (req.method === 'GET' && pathname === '/v1/ops/integrity') {
9041
9163
  const command = parsed.searchParams.get('command') || undefined;
9042
9164
  const baseBranch = parsed.searchParams.get('baseBranch') || undefined;