thumbgate 1.29.1 → 1.29.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/.claude/commands/dashboard.md +11 -1
  2. package/.claude/commands/thumbgate-dashboard.md +23 -8
  3. package/.claude-plugin/plugin.json +1 -1
  4. package/.well-known/mcp/server-card.json +1 -1
  5. package/README.md +61 -1
  6. package/adapters/claude/.mcp.json +2 -2
  7. package/adapters/forge/forge.yaml +3 -3
  8. package/adapters/mcp/server-stdio.js +88 -2
  9. package/adapters/opencode/opencode.json +1 -1
  10. package/commands/dashboard.md +11 -1
  11. package/commands/thumbgate-dashboard.md +23 -8
  12. package/config/agent-outcome-monitor-thresholds.json +63 -0
  13. package/config/evals/agent-outcomes-baseline.json +17 -0
  14. package/config/evals/agent-outcomes-golden.json +412 -0
  15. package/config/evals/prompt-eval-baseline.json +23 -0
  16. package/config/schemas/task-outcome-receipt.schema.json +296 -0
  17. package/openapi/openapi.yaml +235 -0
  18. package/package.json +19 -6
  19. package/public/index.html +4 -2
  20. package/public/numbers.html +2 -2
  21. package/scripts/agent-outcome-eval.js +130 -0
  22. package/scripts/agent-outcome-monitor.js +261 -0
  23. package/scripts/agent-reasoning-traces.js +8 -9
  24. package/scripts/async-job-runner.js +107 -13
  25. package/scripts/durability/step.js +121 -12
  26. package/scripts/gates-engine.js +431 -18
  27. package/scripts/human-escalation.js +265 -0
  28. package/scripts/hybrid-feedback-context.js +93 -50
  29. package/scripts/judge-reward-function.js +30 -18
  30. package/scripts/prompt-eval.js +81 -4
  31. package/scripts/schedule-manager.js +249 -0
  32. package/scripts/task-outcomes.js +425 -0
  33. package/scripts/tool-contract-validator.js +287 -59
  34. package/scripts/tool-registry.js +143 -0
  35. package/src/api/server.js +127 -5
@@ -0,0 +1,296 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://thumbgate.ai/schemas/task-outcome-receipt.schema.json",
4
+ "title": "ThumbGate Task Outcome Receipt",
5
+ "type": "object",
6
+ "additionalProperties": false,
7
+ "required": [
8
+ "taskId",
9
+ "taskType",
10
+ "goal",
11
+ "status",
12
+ "verification",
13
+ "toolCalls",
14
+ "policy",
15
+ "efficiency"
16
+ ],
17
+ "properties": {
18
+ "taskId": {
19
+ "type": "string",
20
+ "minLength": 1,
21
+ "maxLength": 200
22
+ },
23
+ "taskType": {
24
+ "type": "string",
25
+ "minLength": 1,
26
+ "maxLength": 100
27
+ },
28
+ "goal": {
29
+ "type": "string",
30
+ "minLength": 1,
31
+ "maxLength": 2000
32
+ },
33
+ "expectedOutcome": {
34
+ "type": "string",
35
+ "maxLength": 2000
36
+ },
37
+ "status": {
38
+ "type": "string",
39
+ "enum": [
40
+ "completed",
41
+ "failed",
42
+ "partial",
43
+ "escalated"
44
+ ]
45
+ },
46
+ "verification": {
47
+ "type": "object",
48
+ "additionalProperties": false,
49
+ "required": [
50
+ "performed",
51
+ "passed",
52
+ "evidence"
53
+ ],
54
+ "properties": {
55
+ "performed": {
56
+ "type": "boolean"
57
+ },
58
+ "passed": {
59
+ "type": "boolean"
60
+ },
61
+ "verifier": {
62
+ "type": "string",
63
+ "maxLength": 200
64
+ },
65
+ "method": {
66
+ "type": "string",
67
+ "maxLength": 200
68
+ },
69
+ "evidence": {
70
+ "type": "array",
71
+ "maxItems": 100,
72
+ "items": {
73
+ "type": "string",
74
+ "minLength": 1,
75
+ "maxLength": 2000
76
+ }
77
+ },
78
+ "unsupportedClaims": {
79
+ "type": "integer",
80
+ "minimum": 0
81
+ }
82
+ }
83
+ },
84
+ "toolCalls": {
85
+ "type": "array",
86
+ "maxItems": 1000,
87
+ "items": {
88
+ "type": "object",
89
+ "additionalProperties": false,
90
+ "required": [
91
+ "name",
92
+ "contractValid",
93
+ "allowed",
94
+ "succeeded",
95
+ "attempts"
96
+ ],
97
+ "properties": {
98
+ "name": {
99
+ "type": "string",
100
+ "minLength": 1,
101
+ "maxLength": 200
102
+ },
103
+ "contractValid": {
104
+ "type": "boolean"
105
+ },
106
+ "allowed": {
107
+ "type": "boolean"
108
+ },
109
+ "succeeded": {
110
+ "type": "boolean"
111
+ },
112
+ "attempts": {
113
+ "type": "integer",
114
+ "minimum": 1
115
+ },
116
+ "latencyMs": {
117
+ "type": "number",
118
+ "minimum": 0
119
+ },
120
+ "costUsd": {
121
+ "type": "number",
122
+ "minimum": 0
123
+ },
124
+ "sideEffect": {
125
+ "type": "boolean"
126
+ },
127
+ "idempotencyKey": {
128
+ "type": "string",
129
+ "maxLength": 300
130
+ },
131
+ "duplicateSideEffect": {
132
+ "type": "boolean"
133
+ }
134
+ }
135
+ }
136
+ },
137
+ "policy": {
138
+ "type": "object",
139
+ "additionalProperties": false,
140
+ "required": [
141
+ "violations",
142
+ "unsafeEscapes",
143
+ "falseBlocks"
144
+ ],
145
+ "properties": {
146
+ "violations": {
147
+ "type": "integer",
148
+ "minimum": 0
149
+ },
150
+ "unsafeEscapes": {
151
+ "type": "integer",
152
+ "minimum": 0
153
+ },
154
+ "falseBlocks": {
155
+ "type": "integer",
156
+ "minimum": 0
157
+ }
158
+ }
159
+ },
160
+ "failure": {
161
+ "type": "object",
162
+ "additionalProperties": false,
163
+ "properties": {
164
+ "category": {
165
+ "type": "string",
166
+ "maxLength": 200
167
+ },
168
+ "recovered": {
169
+ "type": "boolean"
170
+ },
171
+ "repeated": {
172
+ "type": "boolean"
173
+ },
174
+ "rolledBack": {
175
+ "type": "boolean"
176
+ }
177
+ }
178
+ },
179
+ "escalation": {
180
+ "type": "object",
181
+ "additionalProperties": false,
182
+ "properties": {
183
+ "required": {
184
+ "type": "boolean"
185
+ },
186
+ "correct": {
187
+ "type": "boolean"
188
+ },
189
+ "escalationId": {
190
+ "type": "string",
191
+ "maxLength": 200
192
+ }
193
+ }
194
+ },
195
+ "efficiency": {
196
+ "type": "object",
197
+ "additionalProperties": false,
198
+ "required": [
199
+ "latencyMs",
200
+ "costUsd"
201
+ ],
202
+ "properties": {
203
+ "latencyMs": {
204
+ "type": "number",
205
+ "minimum": 0
206
+ },
207
+ "costUsd": {
208
+ "type": "number",
209
+ "minimum": 0
210
+ },
211
+ "firstAttempt": {
212
+ "type": "boolean"
213
+ }
214
+ }
215
+ },
216
+ "businessOutcome": {
217
+ "type": "object",
218
+ "additionalProperties": false,
219
+ "required": [
220
+ "kpi",
221
+ "value",
222
+ "unit"
223
+ ],
224
+ "properties": {
225
+ "kpi": {
226
+ "type": "string",
227
+ "minLength": 1,
228
+ "maxLength": 200
229
+ },
230
+ "value": {
231
+ "type": "number"
232
+ },
233
+ "unit": {
234
+ "type": "string",
235
+ "minLength": 1,
236
+ "maxLength": 50
237
+ }
238
+ }
239
+ },
240
+ "traceId": {
241
+ "type": "string",
242
+ "maxLength": 200
243
+ },
244
+ "idempotencyKey": {
245
+ "type": "string",
246
+ "maxLength": 300
247
+ },
248
+ "versions": {
249
+ "type": "object",
250
+ "additionalProperties": false,
251
+ "properties": {
252
+ "model": {
253
+ "type": "string",
254
+ "maxLength": 200
255
+ },
256
+ "prompt": {
257
+ "type": "string",
258
+ "maxLength": 200
259
+ },
260
+ "tools": {
261
+ "type": "string",
262
+ "maxLength": 200
263
+ },
264
+ "policy": {
265
+ "type": "string",
266
+ "maxLength": 200
267
+ },
268
+ "release": {
269
+ "type": "string",
270
+ "maxLength": 200
271
+ }
272
+ }
273
+ },
274
+ "metadata": {
275
+ "type": "object",
276
+ "additionalProperties": true
277
+ },
278
+ "recordedAt": {
279
+ "type": "string",
280
+ "format": "date-time"
281
+ },
282
+ "receiptHash": {
283
+ "type": "string",
284
+ "pattern": "^[a-f0-9]{64}$"
285
+ },
286
+ "working": {
287
+ "type": "boolean"
288
+ },
289
+ "workingReasons": {
290
+ "type": "array",
291
+ "items": {
292
+ "type": "string"
293
+ }
294
+ }
295
+ }
296
+ }
@@ -19,6 +19,11 @@ components:
19
19
  type: http
20
20
  scheme: bearer
21
21
  bearerFormat: API Key
22
+ humanReviewerKey:
23
+ type: apiKey
24
+ in: header
25
+ name: X-ThumbGate-Human-Reviewer-Key
26
+ description: Independently revocable credential for the server-configured human reviewer identity.
22
27
  schemas:
23
28
  RubricScore:
24
29
  type: object
@@ -77,6 +82,135 @@ components:
77
82
  enum: [chatgpt_gpt]
78
83
  default: chatgpt_gpt
79
84
  description: Attribution marker for ThumbGate analytics. The published ThumbGate GPT should send `chatgpt_gpt` so owner dashboards can distinguish GPT Action calls from local API calls.
85
+ TaskOutcomeReceipt:
86
+ type: object
87
+ additionalProperties: false
88
+ required: [taskId, goal, status, verification, toolCalls, policy, efficiency]
89
+ properties:
90
+ taskId:
91
+ type: string
92
+ taskType:
93
+ type: string
94
+ goal:
95
+ type: string
96
+ expectedOutcome:
97
+ type: string
98
+ status:
99
+ type: string
100
+ enum: [completed, failed, partial, escalated]
101
+ verification:
102
+ type: object
103
+ required: [performed, passed, evidence]
104
+ properties:
105
+ performed:
106
+ type: boolean
107
+ passed:
108
+ type: boolean
109
+ verifier:
110
+ type: string
111
+ method:
112
+ type: string
113
+ evidence:
114
+ type: array
115
+ items:
116
+ type: string
117
+ unsupportedClaims:
118
+ type: integer
119
+ minimum: 0
120
+ toolCalls:
121
+ type: array
122
+ items:
123
+ type: object
124
+ required: [name, contractValid, allowed, succeeded, attempts]
125
+ properties:
126
+ name:
127
+ type: string
128
+ contractValid:
129
+ type: boolean
130
+ allowed:
131
+ type: boolean
132
+ succeeded:
133
+ type: boolean
134
+ attempts:
135
+ type: integer
136
+ minimum: 1
137
+ idempotencyKey:
138
+ type: string
139
+ duplicateSideEffect:
140
+ type: boolean
141
+ policy:
142
+ type: object
143
+ required: [violations, unsafeEscapes, falseBlocks]
144
+ properties:
145
+ violations:
146
+ type: integer
147
+ unsafeEscapes:
148
+ type: integer
149
+ falseBlocks:
150
+ type: integer
151
+ efficiency:
152
+ type: object
153
+ required: [latencyMs, costUsd]
154
+ properties:
155
+ latencyMs:
156
+ type: number
157
+ costUsd:
158
+ type: number
159
+ firstAttempt:
160
+ type: boolean
161
+ idempotencyKey:
162
+ type: string
163
+ EscalationIdentity:
164
+ type: object
165
+ additionalProperties: false
166
+ required: [id, kind]
167
+ properties:
168
+ id:
169
+ type: string
170
+ minLength: 1
171
+ kind:
172
+ type: string
173
+ enum: [agent, service, human]
174
+ displayName:
175
+ type: string
176
+ HumanEscalationRequest:
177
+ type: object
178
+ additionalProperties: false
179
+ required: [taskId, reason, requester, evidence]
180
+ properties:
181
+ taskId:
182
+ type: string
183
+ minLength: 1
184
+ reason:
185
+ type: string
186
+ minLength: 1
187
+ severity:
188
+ type: string
189
+ enum: [low, medium, high, critical]
190
+ requester:
191
+ $ref: '#/components/schemas/EscalationIdentity'
192
+ evidence:
193
+ type: array
194
+ minItems: 1
195
+ items:
196
+ type: string
197
+ minLength: 1
198
+ ttlMs:
199
+ type: number
200
+ minimum: 1
201
+ idempotencyKey:
202
+ type: string
203
+ HumanEscalationDecision:
204
+ type: object
205
+ additionalProperties: false
206
+ required: [decision, reason]
207
+ properties:
208
+ decision:
209
+ type: string
210
+ enum: [approved, rejected, cancelled]
211
+ reason:
212
+ type: string
213
+ minLength: 1
80
214
  IntentPlanRequest:
81
215
  type: object
82
216
  required: [intentId]
@@ -1940,3 +2074,104 @@ paths:
1940
2074
  description: Invalid PayPal transmission headers, event, or remote signature verification
1941
2075
  '503':
1942
2076
  description: PayPal verified the event but durable evidence storage failed
2077
+ /v1/task-outcomes:
2078
+ post:
2079
+ operationId: recordTaskOutcome
2080
+ requestBody:
2081
+ required: true
2082
+ content:
2083
+ application/json:
2084
+ schema:
2085
+ $ref: '#/components/schemas/TaskOutcomeReceipt'
2086
+ responses:
2087
+ '201':
2088
+ description: Task outcome recorded
2089
+ '400':
2090
+ description: Invalid or conflicting task outcome
2091
+ '401':
2092
+ description: Unauthorized
2093
+ get:
2094
+ operationId: listTaskOutcomes
2095
+ parameters:
2096
+ - in: query
2097
+ name: taskId
2098
+ schema:
2099
+ type: string
2100
+ - in: query
2101
+ name: limit
2102
+ schema:
2103
+ type: integer
2104
+ default: 20
2105
+ responses:
2106
+ '200':
2107
+ description: Task outcomes
2108
+ '401':
2109
+ description: Unauthorized
2110
+ /v1/task-outcomes/metrics:
2111
+ get:
2112
+ operationId: getTaskOutcomeMetrics
2113
+ responses:
2114
+ '200':
2115
+ description: Transparent task, tool, safety, escalation, cost, latency, and business metrics
2116
+ '401':
2117
+ description: Unauthorized
2118
+ /v1/task-outcomes/monitor:
2119
+ get:
2120
+ operationId: monitorTaskOutcomes
2121
+ responses:
2122
+ '200':
2123
+ description: Production task outcome threshold verdict
2124
+ '401':
2125
+ description: Unauthorized
2126
+ /v1/escalations:
2127
+ post:
2128
+ operationId: requestHumanEscalation
2129
+ requestBody:
2130
+ required: true
2131
+ content:
2132
+ application/json:
2133
+ schema:
2134
+ $ref: '#/components/schemas/HumanEscalationRequest'
2135
+ responses:
2136
+ '201':
2137
+ description: Human escalation requested
2138
+ '400':
2139
+ description: Invalid or conflicting request
2140
+ '401':
2141
+ description: Unauthorized
2142
+ get:
2143
+ operationId: listHumanEscalations
2144
+ responses:
2145
+ '200':
2146
+ description: Human escalation queue and metrics
2147
+ '401':
2148
+ description: Unauthorized
2149
+ /v1/escalations/{escalationId}/decision:
2150
+ post:
2151
+ operationId: decideHumanEscalation
2152
+ security:
2153
+ - bearerAuth: []
2154
+ humanReviewerKey: []
2155
+ parameters:
2156
+ - in: path
2157
+ name: escalationId
2158
+ required: true
2159
+ schema:
2160
+ type: string
2161
+ requestBody:
2162
+ required: true
2163
+ content:
2164
+ application/json:
2165
+ schema:
2166
+ $ref: '#/components/schemas/HumanEscalationDecision'
2167
+ responses:
2168
+ '200':
2169
+ description: Human escalation decision recorded
2170
+ '400':
2171
+ description: Invalid decision or the authenticated reviewer matches the requester
2172
+ '401':
2173
+ description: Unauthorized
2174
+ '403':
2175
+ description: Missing or invalid human reviewer credential
2176
+ '503':
2177
+ description: Human reviewer identity is not configured