thumbgate 1.26.0 → 1.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +62 -31
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/mcp/server-stdio.js +83 -6
- package/adapters/opencode/opencode.json +1 -1
- package/bin/cli.js +390 -14
- package/config/mcp-allowlists.json +3 -0
- package/package.json +12 -2
- package/public/agents-cost-savings.html +2 -0
- package/public/index.html +10 -2
- package/public/numbers.html +2 -2
- package/scripts/action-receipts.js +324 -0
- package/scripts/cli-schema.js +24 -0
- package/scripts/dashboard.js +6 -1
- package/scripts/gates-engine.js +28 -9
- package/scripts/llm-client.js +90 -4
- package/scripts/local-model-profile.js +15 -8
- package/scripts/meta-agent-loop.js +9 -5
- package/scripts/noop-detect.js +285 -0
- package/scripts/operational-dashboard.js +160 -0
- package/scripts/plan-gate.js +11 -0
- package/scripts/repeat-metric.js +121 -0
- package/scripts/silent-failure-cluster.js +22 -3
- package/scripts/tool-registry.js +50 -0
|
@@ -377,13 +377,17 @@ async function runMetaAgentLoop({ dryRun = false, verbose = false } = {}) {
|
|
|
377
377
|
// measurement (silentFailureDerivedGates vs user-feedback-derived) is possible.
|
|
378
378
|
candidates = candidates.map((c) => (c.origin ? c : { ...c, origin: 'user-feedback' }));
|
|
379
379
|
|
|
380
|
-
// Step 3b: Silent-failure clustering —
|
|
381
|
-
//
|
|
382
|
-
//
|
|
380
|
+
// Step 3b: Silent-failure clustering — DEFAULT-ON as of 2026-05-21
|
|
381
|
+
// (flipped from opt-in by PR #2289). Opt out via
|
|
382
|
+
// THUMBGATE_SILENT_FAILURE_CLUSTERING=0 (or NODE_ENV=test). Candidates flow
|
|
383
|
+
// through the SAME scoring / fp-rate eval below; we do not bypass any
|
|
384
|
+
// guardrail. The point of this clustering is to cover the case where users
|
|
385
|
+
// are lazy and never give thumbs-down — keeping it opt-in meant the users
|
|
386
|
+
// who need it most never got the benefit.
|
|
383
387
|
let silentFailureStats = null;
|
|
384
|
-
|
|
388
|
+
const { isSilentFailureClusteringEnabled, generateSilentFailureCandidates } = require('./silent-failure-cluster');
|
|
389
|
+
if (isSilentFailureClusteringEnabled()) {
|
|
385
390
|
try {
|
|
386
|
-
const { generateSilentFailureCandidates } = require('./silent-failure-cluster');
|
|
387
391
|
const sfResult = generateSilentFailureCandidates({ feedbackLogPath });
|
|
388
392
|
silentFailureStats = sfResult.stats;
|
|
389
393
|
if (sfResult.candidates && sfResult.candidates.length > 0) {
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// scripts/noop-detect.js
|
|
4
|
+
//
|
|
5
|
+
// No-op / redundant-action detection for ThumbGate.
|
|
6
|
+
//
|
|
7
|
+
// Given an action's pre/post state (file diff, command exit code + output),
|
|
8
|
+
// this module decides whether the action actually changed anything OR whether
|
|
9
|
+
// it is byte-identical to an attempt already seen this session. Both are strong,
|
|
10
|
+
// cheap "the agent is looping" repeat signals that plug into
|
|
11
|
+
// track_action -> prevention_rules.
|
|
12
|
+
//
|
|
13
|
+
// Design goals:
|
|
14
|
+
// * Pure / file-local. No edits to shared modules from inside here.
|
|
15
|
+
// * Normalize volatile fields (timestamps, shas, ANSI, trailing whitespace)
|
|
16
|
+
// before hashing so non-deterministic output does not defeat detection.
|
|
17
|
+
// * Guard partial writes: a truncated after-state that is a strict prefix of
|
|
18
|
+
// the before-state must NOT be reported as a no-op.
|
|
19
|
+
//
|
|
20
|
+
// Persistence lives beside session-actions.json (derived from
|
|
21
|
+
// gates-engine.SESSION_ACTIONS_PATH) so it picks up THUMBGATE_STATE_DIR
|
|
22
|
+
// overrides and shares the same TTL semantics.
|
|
23
|
+
|
|
24
|
+
const crypto = require('crypto');
|
|
25
|
+
const fs = require('fs');
|
|
26
|
+
const path = require('path');
|
|
27
|
+
|
|
28
|
+
const gatesEngine = require('./gates-engine');
|
|
29
|
+
|
|
30
|
+
// Match the same 1h session window the engine uses for session actions.
|
|
31
|
+
const ATTEMPT_TTL_MS = 60 * 60 * 1000;
|
|
32
|
+
|
|
33
|
+
// -- volatile-field normalization ------------------------------------------
|
|
34
|
+
//
|
|
35
|
+
// Each pattern strips a class of non-deterministic noise so that two outputs
|
|
36
|
+
// which differ only by a timestamp / sha / uuid / ANSI color / trailing
|
|
37
|
+
// whitespace hash-equal. Exported so the test suite can assert the contract.
|
|
38
|
+
const VOLATILE_PATTERNS = [
|
|
39
|
+
// ANSI escape / color codes (e.g. \x1b[0m). Strip first so later patterns
|
|
40
|
+
// see clean text.
|
|
41
|
+
{ name: 'ansi', re: /\x1b\[[0-9;]*[A-Za-z]/g, replacement: '' },
|
|
42
|
+
// ISO-8601 timestamps: 2026-05-31T12:34:56(.123)?(Z|+00:00)
|
|
43
|
+
{
|
|
44
|
+
name: 'iso8601',
|
|
45
|
+
re: /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g,
|
|
46
|
+
replacement: '<TS>',
|
|
47
|
+
},
|
|
48
|
+
// UUIDs (dashed form) — must run before the hexblob pattern, otherwise the
|
|
49
|
+
// trailing 12-char hex segment gets rewritten first and the UUID never matches.
|
|
50
|
+
{
|
|
51
|
+
name: 'uuid',
|
|
52
|
+
re: /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g,
|
|
53
|
+
replacement: '<UUID>',
|
|
54
|
+
},
|
|
55
|
+
// Epoch integers wider than 10 digits (ms/ns timestamps) — run before the
|
|
56
|
+
// hexblob pattern so all-decimal epochs are not swallowed as hex first.
|
|
57
|
+
{ name: 'epoch', re: /\b\d{11,}\b/g, replacement: '<EPOCH>' },
|
|
58
|
+
// Long hex blobs (commit shas, uuids without dashes, content hashes): 12+ hex chars.
|
|
59
|
+
{ name: 'hexblob', re: /\b[0-9a-fA-F]{12,}\b/g, replacement: '<HEX>' },
|
|
60
|
+
// Trailing whitespace is stripped in normalizeVolatile() via a bounded linear
|
|
61
|
+
// scan (not a regex) to avoid super-linear backtracking on adversarial input.
|
|
62
|
+
];
|
|
63
|
+
|
|
64
|
+
// Strip trailing spaces/tabs from a single line via a bounded reverse scan.
|
|
65
|
+
// Linear in line length; replaces /[ \t]+$/ without any regex backtracking.
|
|
66
|
+
function stripTrailingSpaceTab(line) {
|
|
67
|
+
let end = line.length;
|
|
68
|
+
while (end > 0) {
|
|
69
|
+
const c = line.charCodeAt(end - 1);
|
|
70
|
+
if (c !== 32 && c !== 9) break; // 32 = space, 9 = tab
|
|
71
|
+
end -= 1;
|
|
72
|
+
}
|
|
73
|
+
return line.slice(0, end);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Normalize a string by applying every volatile pattern, then trimming a
|
|
77
|
+
// trailing newline so "foo\n" and "foo" are equivalent.
|
|
78
|
+
function normalizeVolatile(value) {
|
|
79
|
+
if (value === null || value === undefined) return '';
|
|
80
|
+
let out = String(value);
|
|
81
|
+
for (const pattern of VOLATILE_PATTERNS) {
|
|
82
|
+
out = out.replace(pattern.re, pattern.replacement);
|
|
83
|
+
}
|
|
84
|
+
// Strip trailing space/tab per line without a backtracking-prone regex.
|
|
85
|
+
out = out.split('\n').map(stripTrailingSpaceTab).join('\n');
|
|
86
|
+
// Normalize CRLF, then strip the trailing run of newlines via a bounded
|
|
87
|
+
// linear scan (replaces /\n+$/ without regex backtracking).
|
|
88
|
+
out = out.replace(/\r\n/g, '\n');
|
|
89
|
+
let end = out.length;
|
|
90
|
+
while (end > 0 && out.charCodeAt(end - 1) === 10) end -= 1; // 10 = \n
|
|
91
|
+
return out.slice(0, end);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function sha256(value) {
|
|
95
|
+
return crypto.createHash('sha256').update(value, 'utf8').digest('hex');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// -- state hashing ----------------------------------------------------------
|
|
99
|
+
//
|
|
100
|
+
// computeActionStateHash produces a single stable fingerprint for an action's
|
|
101
|
+
// observable state. For file actions the fingerprint is the normalized content;
|
|
102
|
+
// for command actions it is exit code + normalized stdout/stderr.
|
|
103
|
+
function computeActionStateHash(action = {}) {
|
|
104
|
+
const kind = action.kind === 'command' ? 'command' : 'file';
|
|
105
|
+
|
|
106
|
+
if (kind === 'command') {
|
|
107
|
+
const exitCode = action.exitCode === undefined || action.exitCode === null
|
|
108
|
+
? ''
|
|
109
|
+
: String(action.exitCode);
|
|
110
|
+
const parts = [
|
|
111
|
+
'command',
|
|
112
|
+
`exit:${exitCode}`,
|
|
113
|
+
`stdout:${normalizeVolatile(action.stdout)}`,
|
|
114
|
+
`stderr:${normalizeVolatile(action.stderr)}`,
|
|
115
|
+
];
|
|
116
|
+
return sha256(parts.join('\x00'));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// file kind: hash the relevant content (after-content is the canonical state
|
|
120
|
+
// for a single snapshot; for diff comparisons detectNoop compares before/after
|
|
121
|
+
// separately). Include filePath so two unrelated files never collide.
|
|
122
|
+
const content = action.afterContent !== undefined && action.afterContent !== null
|
|
123
|
+
? action.afterContent
|
|
124
|
+
: action.beforeContent;
|
|
125
|
+
const parts = [
|
|
126
|
+
'file',
|
|
127
|
+
`path:${action.filePath || ''}`,
|
|
128
|
+
`content:${normalizeVolatile(content)}`,
|
|
129
|
+
];
|
|
130
|
+
return sha256(parts.join('\x00'));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Hash just a content blob for before/after comparison (no path/kind framing).
|
|
134
|
+
function contentHash(content) {
|
|
135
|
+
return sha256(normalizeVolatile(content));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// -- no-op detection --------------------------------------------------------
|
|
139
|
+
//
|
|
140
|
+
// detectNoop({ before, after }) returns { noop, reason }.
|
|
141
|
+
// before/after: { kind, filePath, beforeContent/afterContent OR content,
|
|
142
|
+
// exitCode, stdout, stderr }
|
|
143
|
+
//
|
|
144
|
+
// We accept a flexible shape: the caller may pass a single action object with
|
|
145
|
+
// beforeContent/afterContent, or split before/after sub-objects. Both forms
|
|
146
|
+
// resolve to the same decision.
|
|
147
|
+
function detectNoop(input = {}) {
|
|
148
|
+
const before = input.before || {};
|
|
149
|
+
const after = input.after || {};
|
|
150
|
+
|
|
151
|
+
// Determine kind from whichever side declares it (default file).
|
|
152
|
+
const kind = (before.kind || after.kind || input.kind) === 'command'
|
|
153
|
+
? 'command'
|
|
154
|
+
: 'file';
|
|
155
|
+
|
|
156
|
+
if (kind === 'command') {
|
|
157
|
+
const beforeExit = before.exitCode;
|
|
158
|
+
const afterExit = after.exitCode;
|
|
159
|
+
if (beforeExit !== undefined && afterExit !== undefined && beforeExit !== afterExit) {
|
|
160
|
+
return { noop: false, reason: 'exit-code-changed' };
|
|
161
|
+
}
|
|
162
|
+
const beforeOut = contentHash(`${normalizeVolatile(before.stdout)}\x00${normalizeVolatile(before.stderr)}`);
|
|
163
|
+
const afterOut = contentHash(`${normalizeVolatile(after.stdout)}\x00${normalizeVolatile(after.stderr)}`);
|
|
164
|
+
if (beforeOut === afterOut) {
|
|
165
|
+
return { noop: true, reason: 'command-output-unchanged' };
|
|
166
|
+
}
|
|
167
|
+
return { noop: false, reason: 'command-output-changed' };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// file kind.
|
|
171
|
+
const beforeContent = before.content !== undefined ? before.content
|
|
172
|
+
: (before.beforeContent !== undefined ? before.beforeContent : input.beforeContent);
|
|
173
|
+
const afterContent = after.content !== undefined ? after.content
|
|
174
|
+
: (after.afterContent !== undefined ? after.afterContent : input.afterContent);
|
|
175
|
+
|
|
176
|
+
const beforeStr = beforeContent === undefined || beforeContent === null ? '' : String(beforeContent);
|
|
177
|
+
const afterStr = afterContent === undefined || afterContent === null ? '' : String(afterContent);
|
|
178
|
+
|
|
179
|
+
// Partial-write guard: an after-state that is a strict, shorter prefix of the
|
|
180
|
+
// before-state is a truncation, not a no-op — even though hashes differ, we
|
|
181
|
+
// make the intent explicit so callers never mistake it.
|
|
182
|
+
if (afterStr.length > 0 && afterStr.length < beforeStr.length && beforeStr.startsWith(afterStr)) {
|
|
183
|
+
return { noop: false, reason: 'partial-write-truncation' };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (contentHash(beforeStr) === contentHash(afterStr)) {
|
|
187
|
+
return { noop: true, reason: 'file-content-unchanged' };
|
|
188
|
+
}
|
|
189
|
+
return { noop: false, reason: 'file-content-changed' };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// -- attempt persistence ----------------------------------------------------
|
|
193
|
+
//
|
|
194
|
+
// Stores (sessionId -> { "actionId\x00stateHash": timestamp }) so a repeated
|
|
195
|
+
// (actionId, stateHash) tuple within the TTL window is a strong repeat signal.
|
|
196
|
+
function attemptsPath() {
|
|
197
|
+
const sessionActionsPath = gatesEngine.SESSION_ACTIONS_PATH;
|
|
198
|
+
return path.join(path.dirname(sessionActionsPath), 'noop-attempts.json');
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function loadAttempts() {
|
|
202
|
+
try {
|
|
203
|
+
const raw = fs.readFileSync(attemptsPath(), 'utf8');
|
|
204
|
+
const parsed = JSON.parse(raw);
|
|
205
|
+
if (!parsed || typeof parsed !== 'object') return {};
|
|
206
|
+
return parsed;
|
|
207
|
+
} catch (e) {
|
|
208
|
+
return {};
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function pruneExpired(store, now) {
|
|
213
|
+
let changed = false;
|
|
214
|
+
for (const sessionId of Object.keys(store)) {
|
|
215
|
+
const bucket = store[sessionId];
|
|
216
|
+
if (!bucket || typeof bucket !== 'object') {
|
|
217
|
+
delete store[sessionId];
|
|
218
|
+
changed = true;
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
for (const key of Object.keys(bucket)) {
|
|
222
|
+
const ts = bucket[key];
|
|
223
|
+
if (typeof ts !== 'number' || now - ts >= ATTEMPT_TTL_MS) {
|
|
224
|
+
delete bucket[key];
|
|
225
|
+
changed = true;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (Object.keys(bucket).length === 0) {
|
|
229
|
+
delete store[sessionId];
|
|
230
|
+
changed = true;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return changed;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function saveAttempts(store) {
|
|
237
|
+
const file = attemptsPath();
|
|
238
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
239
|
+
fs.writeFileSync(file, JSON.stringify(store, null, 2) + '\n');
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function attemptKey(actionId, stateHash) {
|
|
243
|
+
return `${String(actionId)}\x00${String(stateHash)}`;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// recordActionAttempt persists that (sessionId, actionId, stateHash) was seen.
|
|
247
|
+
// Returns { recorded: boolean, alreadySeen: boolean }.
|
|
248
|
+
function recordActionAttempt(sessionId, actionId, stateHash) {
|
|
249
|
+
const sid = String(sessionId || 'default');
|
|
250
|
+
const now = Date.now();
|
|
251
|
+
const store = loadAttempts();
|
|
252
|
+
pruneExpired(store, now);
|
|
253
|
+
|
|
254
|
+
if (!store[sid] || typeof store[sid] !== 'object') store[sid] = {};
|
|
255
|
+
const key = attemptKey(actionId, stateHash);
|
|
256
|
+
const alreadySeen = Object.prototype.hasOwnProperty.call(store[sid], key);
|
|
257
|
+
store[sid][key] = now;
|
|
258
|
+
saveAttempts(store);
|
|
259
|
+
return { recorded: true, alreadySeen };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// isRepeatAttempt returns true when this exact (actionId, stateHash) tuple was
|
|
263
|
+
// already recorded for this session within the TTL window.
|
|
264
|
+
function isRepeatAttempt(sessionId, actionId, stateHash) {
|
|
265
|
+
const sid = String(sessionId || 'default');
|
|
266
|
+
const now = Date.now();
|
|
267
|
+
const store = loadAttempts();
|
|
268
|
+
const bucket = store[sid];
|
|
269
|
+
if (!bucket || typeof bucket !== 'object') return false;
|
|
270
|
+
const ts = bucket[attemptKey(actionId, stateHash)];
|
|
271
|
+
if (typeof ts !== 'number') return false;
|
|
272
|
+
return now - ts < ATTEMPT_TTL_MS;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
module.exports = {
|
|
276
|
+
VOLATILE_PATTERNS,
|
|
277
|
+
ATTEMPT_TTL_MS,
|
|
278
|
+
normalizeVolatile,
|
|
279
|
+
computeActionStateHash,
|
|
280
|
+
detectNoop,
|
|
281
|
+
recordActionAttempt,
|
|
282
|
+
isRepeatAttempt,
|
|
283
|
+
// Exposed for tests / introspection.
|
|
284
|
+
attemptsPath,
|
|
285
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { resolveAnalyticsWindow } = require('./analytics-window');
|
|
4
|
+
const { getBillingSummaryLive } = require('./billing');
|
|
5
|
+
const { generateDashboard } = require('./dashboard');
|
|
6
|
+
const { getFeedbackPaths } = require('./feedback-loop');
|
|
7
|
+
const { resolveHostedBillingConfig } = require('./hosted-config');
|
|
8
|
+
const { loadOperatorConfig } = require('./operational-summary');
|
|
9
|
+
|
|
10
|
+
function normalizeText(value) {
|
|
11
|
+
if (value === undefined || value === null) return null;
|
|
12
|
+
const text = String(value).trim();
|
|
13
|
+
return text || null;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function shouldPreferHostedDashboard() {
|
|
17
|
+
return String(process.env.THUMBGATE_METRICS_SOURCE || '').trim().toLowerCase() !== 'local';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function resolveHostedDashboardConfig() {
|
|
21
|
+
const runtimeConfig = resolveHostedBillingConfig();
|
|
22
|
+
const operatorConfig = loadOperatorConfig();
|
|
23
|
+
// Match operational-summary's key priority chain so north-star and cfo
|
|
24
|
+
// authenticate against the same hosted deployment consistently. Prior to
|
|
25
|
+
// this change, north-star only read THUMBGATE_API_KEY, silently 401'ing
|
|
26
|
+
// on machines configured via operator.json or THUMBGATE_OPERATOR_KEY.
|
|
27
|
+
const apiKey = normalizeText(process.env.THUMBGATE_OPERATOR_KEY)
|
|
28
|
+
|| operatorConfig.operatorKey
|
|
29
|
+
|| normalizeText(process.env.THUMBGATE_API_KEY);
|
|
30
|
+
const apiBaseUrl = normalizeText(process.env.THUMBGATE_BILLING_API_BASE_URL)
|
|
31
|
+
|| operatorConfig.baseUrl
|
|
32
|
+
|| runtimeConfig.billingApiBaseUrl;
|
|
33
|
+
return {
|
|
34
|
+
apiBaseUrl,
|
|
35
|
+
apiKey,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function buildOperationalDashboard(options = {}) {
|
|
40
|
+
const analyticsWindow = resolveAnalyticsWindow(options);
|
|
41
|
+
const feedbackDir = options.feedbackDir || getFeedbackPaths().FEEDBACK_DIR;
|
|
42
|
+
const billingSummary = await getBillingSummaryLive(analyticsWindow);
|
|
43
|
+
|
|
44
|
+
return generateDashboard(feedbackDir, {
|
|
45
|
+
analyticsWindow,
|
|
46
|
+
billingSummary,
|
|
47
|
+
billingSource: 'live',
|
|
48
|
+
billingFallbackReason: null,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function fetchHostedDashboard(options = {}, config = resolveHostedDashboardConfig()) {
|
|
53
|
+
const analyticsWindow = resolveAnalyticsWindow(options);
|
|
54
|
+
if (!shouldPreferHostedDashboard()) {
|
|
55
|
+
const err = new Error('Hosted operational dashboard is disabled.');
|
|
56
|
+
err.code = 'hosted_dashboard_disabled';
|
|
57
|
+
throw err;
|
|
58
|
+
}
|
|
59
|
+
if (!config.apiBaseUrl || !config.apiKey) {
|
|
60
|
+
const err = new Error('Hosted operational dashboard is not configured.');
|
|
61
|
+
err.code = 'hosted_dashboard_unconfigured';
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const requestUrl = new URL('/v1/dashboard', config.apiBaseUrl);
|
|
66
|
+
requestUrl.searchParams.set('window', analyticsWindow.window);
|
|
67
|
+
requestUrl.searchParams.set('timezone', analyticsWindow.timeZone);
|
|
68
|
+
requestUrl.searchParams.set('now', analyticsWindow.now);
|
|
69
|
+
|
|
70
|
+
const response = await fetch(requestUrl, {
|
|
71
|
+
method: 'GET',
|
|
72
|
+
headers: {
|
|
73
|
+
authorization: `Bearer ${config.apiKey}`,
|
|
74
|
+
accept: 'application/json',
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
if (!response.ok) {
|
|
79
|
+
const detail = await response.text().catch(() => '');
|
|
80
|
+
const err = new Error(`Hosted operational dashboard request failed (${response.status}): ${detail || 'unknown error'}`);
|
|
81
|
+
err.code = 'hosted_dashboard_http_error';
|
|
82
|
+
err.status = response.status;
|
|
83
|
+
throw err;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return response.json();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async function getOperationalDashboard(options = {}) {
|
|
90
|
+
const analyticsWindow = resolveAnalyticsWindow(options);
|
|
91
|
+
try {
|
|
92
|
+
const data = await fetchHostedDashboard(analyticsWindow);
|
|
93
|
+
return {
|
|
94
|
+
source: 'hosted',
|
|
95
|
+
data,
|
|
96
|
+
fallbackReason: null,
|
|
97
|
+
hostedStatus: 200,
|
|
98
|
+
};
|
|
99
|
+
} catch (err) {
|
|
100
|
+
const reason = err && err.message ? err.message : 'hosted_dashboard_unavailable';
|
|
101
|
+
const status = err && typeof err.status === 'number' ? err.status : null;
|
|
102
|
+
const code = err && err.code ? err.code : null;
|
|
103
|
+
|
|
104
|
+
// Hosted deliberately disabled or never configured — local fallback is
|
|
105
|
+
// intentional, not a degraded state. Tag as plain 'local'.
|
|
106
|
+
if (code === 'hosted_dashboard_disabled' || code === 'hosted_dashboard_unconfigured') {
|
|
107
|
+
return {
|
|
108
|
+
source: 'local',
|
|
109
|
+
data: await buildOperationalDashboard(analyticsWindow),
|
|
110
|
+
fallbackReason: reason,
|
|
111
|
+
hostedStatus: null,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Mirror operational-summary: auth failure is the dangerous case. A
|
|
116
|
+
// dashboard that silently shows $0 revenue (from the local ledger) when
|
|
117
|
+
// Stripe actually has paid customers is a lie the operator acts on.
|
|
118
|
+
// Refuse to guess — surface an actionable error.
|
|
119
|
+
if (status === 401 || status === 403) {
|
|
120
|
+
const authErr = new Error(
|
|
121
|
+
`Hosted operational dashboard rejected credentials (HTTP ${status}). ` +
|
|
122
|
+
`The operator key on this machine does not match the one on the ` +
|
|
123
|
+
`hosted deployment. Fix: set THUMBGATE_OPERATOR_KEY in this shell, ` +
|
|
124
|
+
`or update the operatorKey field in ~/.config/thumbgate/operator.json, ` +
|
|
125
|
+
`to match Railway's THUMBGATE_OPERATOR_KEY. ` +
|
|
126
|
+
`Running north-star without hosted auth would report local-only ` +
|
|
127
|
+
`data as ground truth, which may not reflect actual Stripe revenue. ` +
|
|
128
|
+
`Original response: ${reason}`
|
|
129
|
+
);
|
|
130
|
+
authErr.code = 'hosted_dashboard_unauthorized';
|
|
131
|
+
authErr.status = status;
|
|
132
|
+
throw authErr;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Non-auth failure — local fallback is still useful for dev workflows,
|
|
136
|
+
// but tag the source so downstream renderers do not mistake it for
|
|
137
|
+
// verified hosted truth.
|
|
138
|
+
//
|
|
139
|
+
// Log only the status code (trusted) — the full reason contains upstream
|
|
140
|
+
// response text and is only returned structurally via fallbackReason.
|
|
141
|
+
console.warn(
|
|
142
|
+
`[operational-dashboard] Hosted dashboard unreachable (status=${status ?? 'network'}); ` +
|
|
143
|
+
`falling back to LOCAL-UNVERIFIED state. Numbers below may not reflect actual Stripe revenue.`
|
|
144
|
+
);
|
|
145
|
+
return {
|
|
146
|
+
source: 'local-unverified',
|
|
147
|
+
data: await buildOperationalDashboard(analyticsWindow),
|
|
148
|
+
fallbackReason: reason,
|
|
149
|
+
hostedStatus: status,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
module.exports = {
|
|
155
|
+
buildOperationalDashboard,
|
|
156
|
+
fetchHostedDashboard,
|
|
157
|
+
getOperationalDashboard,
|
|
158
|
+
resolveHostedDashboardConfig,
|
|
159
|
+
shouldPreferHostedDashboard,
|
|
160
|
+
};
|
package/scripts/plan-gate.js
CHANGED
|
@@ -162,6 +162,17 @@ function evaluatePlanGate(toolName, toolInput, options = {}) {
|
|
|
162
162
|
};
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
// Tier 4: Self-Critique / Risk Mitigation Check (Tip #8)
|
|
166
|
+
const hasCritique = /(?:critique|self-critique|risk|mitigation|alternative|flaw|weakness|pitfall)/i.test(planContent);
|
|
167
|
+
if (!hasCritique) {
|
|
168
|
+
return {
|
|
169
|
+
decision: 'warn',
|
|
170
|
+
gate: 'plan-gate-critique-missing',
|
|
171
|
+
message: '🧐 THUMBGATE: No Self-Critique/Risk Analysis detected in your PLAN.md. Please add a "Critique", "Risks", or "Mitigations" section to evaluate potential flaws in this plan before executing high-risk tools.',
|
|
172
|
+
severity: 'medium'
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
165
176
|
return null;
|
|
166
177
|
}
|
|
167
178
|
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// repeat-metric — first-class "repeat-attempts blocked before execution" metric
|
|
5
|
+
//
|
|
6
|
+
// This module exposes data ThumbGate already collects in gate-stats state. It
|
|
7
|
+
// does NOT write to disk; it is a pure function over gates-engine.loadStats().
|
|
8
|
+
//
|
|
9
|
+
// The headline number is stats.recurringBlocks — incremented by recordStat()
|
|
10
|
+
// in gates-engine.js every time the SAME gateId fires twice within one session
|
|
11
|
+
// bucket. That is exactly "a pre-action gate fire that stopped a tool call the
|
|
12
|
+
// agent had already been blocked on", i.e. a repeat attempt prevented before it
|
|
13
|
+
// could round-trip and execute.
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
const gatesEngine = require('./gates-engine');
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Derive a per-gate { firstBlocks, repeatBlocks } split from the raw stats.
|
|
20
|
+
*
|
|
21
|
+
* recordStat() records, per session bucket, which gates have fired
|
|
22
|
+
* (stats.sessionFiredGates[sessionKey][gateId] === true). The FIRST fire of a
|
|
23
|
+
* gate in a bucket marks the flag; every subsequent fire in that same bucket
|
|
24
|
+
* increments stats.recurringBlocks. So for each gate:
|
|
25
|
+
* firstBlocks = number of distinct session buckets the gate fired in
|
|
26
|
+
* repeatBlocks = (total block+warn events for the gate) - firstBlocks
|
|
27
|
+
*
|
|
28
|
+
* total block+warn events come from stats.byGate[id] (blocked + warned), which
|
|
29
|
+
* recordStat() also maintains. repeatBlocks is clamped to >= 0 to stay robust
|
|
30
|
+
* against partially-written / legacy state.
|
|
31
|
+
*
|
|
32
|
+
* @param {object} stats raw object returned by gates-engine.loadStats()
|
|
33
|
+
* @returns {Object<string,{firstBlocks:number, repeatBlocks:number}>}
|
|
34
|
+
*/
|
|
35
|
+
function computeByGateSplit(stats) {
|
|
36
|
+
const byGate = {};
|
|
37
|
+
const sessionFiredGates = (stats && stats.sessionFiredGates) || {};
|
|
38
|
+
const rawByGate = (stats && stats.byGate) || {};
|
|
39
|
+
|
|
40
|
+
// Count distinct session buckets each gate fired in => firstBlocks.
|
|
41
|
+
const firstBlocksByGate = {};
|
|
42
|
+
for (const sessionKey of Object.keys(sessionFiredGates)) {
|
|
43
|
+
const fired = sessionFiredGates[sessionKey] || {};
|
|
44
|
+
for (const gateId of Object.keys(fired)) {
|
|
45
|
+
if (fired[gateId]) {
|
|
46
|
+
firstBlocksByGate[gateId] = (firstBlocksByGate[gateId] || 0) + 1;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Union of every gate id we know about from either source.
|
|
52
|
+
const gateIds = new Set([
|
|
53
|
+
...Object.keys(rawByGate),
|
|
54
|
+
...Object.keys(firstBlocksByGate),
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
for (const gateId of gateIds) {
|
|
58
|
+
const gateStat = rawByGate[gateId] || {};
|
|
59
|
+
const totalFires = (gateStat.blocked || 0) + (gateStat.warned || 0);
|
|
60
|
+
const firstBlocks = firstBlocksByGate[gateId] || 0;
|
|
61
|
+
// Repeat fires are total fires beyond the first fire per session bucket.
|
|
62
|
+
const repeatBlocks = Math.max(0, totalFires - firstBlocks);
|
|
63
|
+
byGate[gateId] = { firstBlocks, repeatBlocks };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return byGate;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Compute the repeat-attempts-blocked-before-execution metric.
|
|
71
|
+
*
|
|
72
|
+
* Pure read of gates-engine.loadStats(); no disk writes.
|
|
73
|
+
*
|
|
74
|
+
* @returns {{
|
|
75
|
+
* repeatBlocksBeforeExecution: number,
|
|
76
|
+
* recurringBlocks: number,
|
|
77
|
+
* totalBlocked: number,
|
|
78
|
+
* byGate: Object<string,{firstBlocks:number, repeatBlocks:number}>
|
|
79
|
+
* }}
|
|
80
|
+
*/
|
|
81
|
+
function computeRepeatMetric() {
|
|
82
|
+
let stats;
|
|
83
|
+
try {
|
|
84
|
+
stats = gatesEngine.loadStats() || {};
|
|
85
|
+
} catch (_) {
|
|
86
|
+
stats = {};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const recurringBlocks = Number(stats.recurringBlocks || 0);
|
|
90
|
+
const totalBlocked = Number(stats.blocked || 0);
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
// Headline: a pre-action block that stopped a tool call the agent had
|
|
94
|
+
// already been blocked on this session.
|
|
95
|
+
repeatBlocksBeforeExecution: recurringBlocks,
|
|
96
|
+
recurringBlocks,
|
|
97
|
+
totalBlocked,
|
|
98
|
+
byGate: computeByGateSplit(stats),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Add a `repeat` sub-key to a gate-stats object WITHOUT mutating the original.
|
|
104
|
+
*
|
|
105
|
+
* Takes the object returned by gate-stats.calculateStats() or
|
|
106
|
+
* dashboard.computeGateStats() and returns a shallow copy with the repeat
|
|
107
|
+
* metric attached. The caller's file does not need to import any internals.
|
|
108
|
+
*
|
|
109
|
+
* @param {object} gateStatsObject
|
|
110
|
+
* @returns {object} copy of gateStatsObject with `.repeat`
|
|
111
|
+
*/
|
|
112
|
+
function mergeRepeatMetricIntoGateStats(gateStatsObject) {
|
|
113
|
+
const base = gateStatsObject && typeof gateStatsObject === 'object' ? gateStatsObject : {};
|
|
114
|
+
return Object.assign({}, base, { repeat: computeRepeatMetric() });
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
module.exports = {
|
|
118
|
+
computeRepeatMetric,
|
|
119
|
+
mergeRepeatMetricIntoGateStats,
|
|
120
|
+
computeByGateSplit,
|
|
121
|
+
};
|
|
@@ -4,7 +4,14 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* Silent-Failure Clustering — Unsupervised candidate source for the meta-agent loop
|
|
6
6
|
*
|
|
7
|
-
*
|
|
7
|
+
* Default-ON since 2026-05-21. Opt-out with: THUMBGATE_SILENT_FAILURE_CLUSTERING=0
|
|
8
|
+
* (or set NODE_ENV=test to skip in test runs). Was opt-in for the initial
|
|
9
|
+
* landing of PR #2285; flipped to default-on because the entire point is to
|
|
10
|
+
* cover the case where users never give thumbs-down — keeping it opt-in
|
|
11
|
+
* means lazy users (the ones who need it most) never benefit. Bounded risk:
|
|
12
|
+
* candidates still flow through meta-agent-loop's existing fp-rate eval, so
|
|
13
|
+
* a noisy cluster can't auto-promote to a real gate without passing the
|
|
14
|
+
* same precision/recall thresholds as LLM-generated candidates.
|
|
8
15
|
*
|
|
9
16
|
* Problem: ThumbGate's HITL loop only learns from explicit thumbs-down. Tool calls
|
|
10
17
|
* that fail without user feedback (exit_code != 0, regex-matched error in output,
|
|
@@ -460,9 +467,20 @@ function generateSilentFailureCandidates(opts = {}) {
|
|
|
460
467
|
// CLI
|
|
461
468
|
// ---------------------------------------------------------------------------
|
|
462
469
|
|
|
470
|
+
/**
|
|
471
|
+
* Resolve the enabled state. Default ON. Explicit "0" or "false" opts out;
|
|
472
|
+
* NODE_ENV=test also opts out to keep test runs deterministic.
|
|
473
|
+
*/
|
|
474
|
+
function isSilentFailureClusteringEnabled(env = process.env) {
|
|
475
|
+
if (env.NODE_ENV === 'test') return false;
|
|
476
|
+
const raw = (env.THUMBGATE_SILENT_FAILURE_CLUSTERING || '').toLowerCase();
|
|
477
|
+
if (raw === '0' || raw === 'false' || raw === 'off' || raw === 'no') return false;
|
|
478
|
+
return true;
|
|
479
|
+
}
|
|
480
|
+
|
|
463
481
|
async function main() {
|
|
464
|
-
if (
|
|
465
|
-
process.stdout.write('silent-failure-cluster: disabled (
|
|
482
|
+
if (!isSilentFailureClusteringEnabled()) {
|
|
483
|
+
process.stdout.write('silent-failure-cluster: disabled (THUMBGATE_SILENT_FAILURE_CLUSTERING=0 or NODE_ENV=test)\n');
|
|
466
484
|
return;
|
|
467
485
|
}
|
|
468
486
|
|
|
@@ -492,6 +510,7 @@ if (require.main === module) {
|
|
|
492
510
|
|
|
493
511
|
module.exports = {
|
|
494
512
|
generateSilentFailureCandidates,
|
|
513
|
+
isSilentFailureClusteringEnabled,
|
|
495
514
|
// exported for testing
|
|
496
515
|
redactSecrets,
|
|
497
516
|
normalizePaths,
|