thumbgate 1.28.4 → 1.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/dashboard.md +11 -1
- package/.claude/commands/thumbgate-dashboard.md +23 -8
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/llms.txt +18 -10
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +66 -3
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +88 -2
- package/adapters/opencode/opencode.json +1 -1
- package/bin/cli.js +8 -8
- package/bin/postinstall.js +4 -13
- package/commands/dashboard.md +11 -1
- package/commands/thumbgate-dashboard.md +23 -8
- package/config/agent-outcome-monitor-thresholds.json +63 -0
- package/config/evals/agent-outcomes-baseline.json +17 -0
- package/config/evals/agent-outcomes-golden.json +412 -0
- package/config/evals/prompt-eval-baseline.json +23 -0
- package/config/github-about.json +5 -4
- package/config/post-deploy-marketing-pages.json +6 -6
- package/config/schemas/task-outcome-receipt.schema.json +296 -0
- package/docs/integrations/grafana/README.md +109 -0
- package/docs/integrations/grafana/thumbgate-revenue-evidence-dashboard.json +1930 -0
- package/openapi/openapi.yaml +475 -5
- package/package.json +75 -22
- package/public/agent-manager.html +10 -11
- package/public/agents-cost-savings.html +2 -2
- package/public/assets/brand/thumbgate-logo-transparent.svg +6 -11
- package/public/assets/brand/thumbgate-mark-inline-v3.svg +11 -10
- package/public/assets/brand/thumbgate-mark.svg +10 -11
- package/public/blog/inside-your-boundary.html +114 -0
- package/public/blog/process-over-outcome-gates.html +119 -0
- package/public/blog.html +296 -402
- package/public/brand/thumbgate-mark.svg +5 -9
- package/public/codex-enterprise.html +2 -2
- package/public/compare.html +12 -3
- package/public/diagnostic.html +79 -29
- package/public/guide.html +4 -4
- package/public/index.html +1090 -2098
- package/public/install.html +3 -3
- package/public/js/buyer-intent.js +33 -18
- package/public/numbers.html +2 -2
- package/public/pricing.html +268 -408
- package/public/pro.html +4 -4
- package/scripts/agent-outcome-eval.js +130 -0
- package/scripts/agent-outcome-monitor.js +261 -0
- package/scripts/agent-reasoning-traces.js +8 -9
- package/scripts/async-job-runner.js +107 -13
- package/scripts/billing.js +456 -126
- package/scripts/buyer-paths.js +102 -0
- package/scripts/cli-feedback.js +2 -2
- package/scripts/commercial-offer.js +18 -10
- package/scripts/durability/step.js +121 -12
- package/scripts/external-customer-audit.js +881 -0
- package/scripts/feedback-loop.js +26 -0
- package/scripts/gates-engine.js +554 -19
- package/scripts/grafana-revenue-evidence.js +856 -0
- package/scripts/human-escalation.js +265 -0
- package/scripts/hybrid-feedback-context.js +93 -50
- package/scripts/jsonl-window.js +89 -0
- package/scripts/judge-reward-function.js +30 -18
- package/scripts/lesson-embedding-index.js +3 -7
- package/scripts/meta-agent-loop.js +20 -2
- package/scripts/observability-env.js +139 -0
- package/scripts/observability-setup.js +55 -0
- package/scripts/plausible-domain-config.js +4 -0
- package/scripts/prompt-eval.js +81 -4
- package/scripts/provider-live-evidence.js +1290 -0
- package/scripts/provider-payment-reconciler.js +442 -0
- package/scripts/provider-revenue-evidence.js +249 -0
- package/scripts/rate-limiter.js +1 -5
- package/scripts/revenue-action-eligibility.js +414 -0
- package/scripts/revenue-evidence-remediation.js +694 -0
- package/scripts/revenue-offer-system.js +709 -0
- package/scripts/sales-pipeline.js +1117 -0
- package/scripts/schedule-manager.js +249 -0
- package/scripts/seo-gsd.js +8 -4
- package/scripts/stripe-credentials.js +37 -0
- package/scripts/stripe-revenue-catalog-audit.js +363 -0
- package/scripts/stripe-revenue-catalog.js +164 -0
- package/scripts/task-outcomes.js +425 -0
- package/scripts/telemetry-analytics.js +23 -3
- package/scripts/tool-contract-validator.js +287 -59
- package/scripts/tool-registry.js +143 -0
- package/scripts/vector-store.js +83 -7
- package/scripts/workflow-intake-queue.js +483 -0
- package/src/api/server.js +647 -118
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Human Escalation Queue
|
|
6
|
+
*
|
|
7
|
+
* Agents may request or inspect escalation, but approval decisions require an
|
|
8
|
+
* explicit human actor identity distinct from the requesting agent. Events are
|
|
9
|
+
* append-only so every transition remains auditable.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const crypto = require('node:crypto');
|
|
13
|
+
const fs = require('node:fs');
|
|
14
|
+
const path = require('node:path');
|
|
15
|
+
const { getFeedbackPaths } = require('./feedback-paths');
|
|
16
|
+
|
|
17
|
+
const ESCALATIONS_FILE = 'human-escalations.jsonl';
|
|
18
|
+
const MAX_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
19
|
+
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
20
|
+
const SEVERITIES = new Set(['low', 'medium', 'high', 'critical']);
|
|
21
|
+
const DECISIONS = new Set(['approved', 'rejected', 'cancelled']);
|
|
22
|
+
|
|
23
|
+
function getEscalationsPath(options = {}) {
|
|
24
|
+
return path.join(getFeedbackPaths(options).FEEDBACK_DIR, ESCALATIONS_FILE);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function requestEscalation(input = {}, options = {}) {
|
|
28
|
+
const now = options.now || new Date();
|
|
29
|
+
const taskId = requiredString(input.taskId, 'taskId');
|
|
30
|
+
const reason = requiredString(input.reason, 'reason');
|
|
31
|
+
const requester = requiredIdentity(input.requester, 'requester');
|
|
32
|
+
const evidence = stringArray(input.evidence);
|
|
33
|
+
if (evidence.length === 0) throw escalationError('evidence must contain at least one item');
|
|
34
|
+
const severity = input.severity || 'medium';
|
|
35
|
+
if (!SEVERITIES.has(severity)) throw escalationError(`severity must be one of ${Array.from(SEVERITIES).join(', ')}`);
|
|
36
|
+
const ttlMs = Math.min(MAX_TTL_MS, Math.max(1, finiteNumber(input.ttlMs, DEFAULT_TTL_MS)));
|
|
37
|
+
const idempotencyKey = requiredString(input.idempotencyKey || taskId, 'idempotencyKey');
|
|
38
|
+
const existing = listEscalations(options).find((entry) => entry.idempotencyKey === idempotencyKey);
|
|
39
|
+
|
|
40
|
+
const request = {
|
|
41
|
+
escalationId: input.escalationId || `esc_${crypto.randomUUID()}`,
|
|
42
|
+
idempotencyKey,
|
|
43
|
+
taskId,
|
|
44
|
+
reason,
|
|
45
|
+
severity,
|
|
46
|
+
requester,
|
|
47
|
+
evidence,
|
|
48
|
+
requestedAt: now.toISOString(),
|
|
49
|
+
expiresAt: new Date(now.getTime() + ttlMs).toISOString(),
|
|
50
|
+
status: 'pending',
|
|
51
|
+
eventType: 'requested',
|
|
52
|
+
};
|
|
53
|
+
request.eventHash = eventHash(request);
|
|
54
|
+
|
|
55
|
+
if (existing) {
|
|
56
|
+
if (eventComparableHash(existing) !== eventComparableHash(request)) {
|
|
57
|
+
const error = escalationError(`conflicting request for idempotency key '${idempotencyKey}'`);
|
|
58
|
+
error.code = 'THUMBGATE_IDEMPOTENCY_CONFLICT';
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
return { recorded: false, duplicate: true, escalation: existing };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
appendEvent(request, options);
|
|
65
|
+
return { recorded: true, duplicate: false, escalation: request };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function decideEscalation(input = {}, options = {}) {
|
|
69
|
+
const escalationId = requiredString(input.escalationId, 'escalationId');
|
|
70
|
+
const decision = requiredString(input.decision, 'decision');
|
|
71
|
+
if (!DECISIONS.has(decision)) throw escalationError(`decision must be one of ${Array.from(DECISIONS).join(', ')}`);
|
|
72
|
+
if (Object.hasOwn(input, 'actor')) {
|
|
73
|
+
throw escalationError('decision actor is derived from the authenticated reviewer and must not be supplied by the caller');
|
|
74
|
+
}
|
|
75
|
+
const actor = requiredIdentity(options.authenticatedActor, 'authenticatedActor');
|
|
76
|
+
if (actor.kind !== 'human') throw escalationError('authenticatedActor.kind must be human');
|
|
77
|
+
const reason = requiredString(input.reason, 'reason');
|
|
78
|
+
const current = getEscalation(escalationId, options);
|
|
79
|
+
if (!current) throw escalationError(`unknown escalation '${escalationId}'`);
|
|
80
|
+
if (current.status !== 'pending') throw escalationError(`escalation '${escalationId}' is already ${current.status}`);
|
|
81
|
+
if (sameIdentity(current.requester, actor)) throw escalationError('requester cannot decide their own escalation');
|
|
82
|
+
|
|
83
|
+
const now = options.now || new Date();
|
|
84
|
+
const event = {
|
|
85
|
+
escalationId,
|
|
86
|
+
taskId: current.taskId,
|
|
87
|
+
status: decision,
|
|
88
|
+
eventType: 'decided',
|
|
89
|
+
decision,
|
|
90
|
+
actor,
|
|
91
|
+
reason,
|
|
92
|
+
decidedAt: now.toISOString(),
|
|
93
|
+
};
|
|
94
|
+
event.eventHash = eventHash(event);
|
|
95
|
+
appendEvent(event, options);
|
|
96
|
+
return { recorded: true, escalation: { ...current, ...event } };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function listEscalations(options = {}) {
|
|
100
|
+
const events = readEvents(options);
|
|
101
|
+
const byId = new Map();
|
|
102
|
+
for (const event of events) {
|
|
103
|
+
const current = byId.get(event.escalationId) || {};
|
|
104
|
+
byId.set(event.escalationId, { ...current, ...event });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const nowMs = (options.now || new Date()).getTime();
|
|
108
|
+
const rows = Array.from(byId.values()).map((entry) => {
|
|
109
|
+
if (entry.status === 'pending' && Date.parse(entry.expiresAt) <= nowMs) {
|
|
110
|
+
return { ...entry, status: 'expired' };
|
|
111
|
+
}
|
|
112
|
+
return entry;
|
|
113
|
+
});
|
|
114
|
+
const status = options.status;
|
|
115
|
+
return rows
|
|
116
|
+
.filter((entry) => !status || entry.status === status)
|
|
117
|
+
.sort((a, b) => Date.parse(b.requestedAt || b.decidedAt) - Date.parse(a.requestedAt || a.decidedAt));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function getEscalation(escalationId, options = {}) {
|
|
121
|
+
return listEscalations(options).find((entry) => entry.escalationId === escalationId) || null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function calculateEscalationMetrics(escalations = [], now = new Date()) {
|
|
125
|
+
const rows = escalations.filter(Boolean);
|
|
126
|
+
const decided = rows.filter((entry) => ['approved', 'rejected'].includes(entry.status));
|
|
127
|
+
const decisionLatencies = decided
|
|
128
|
+
.map((entry) => Date.parse(entry.decidedAt) - Date.parse(entry.requestedAt))
|
|
129
|
+
.filter((value) => Number.isFinite(value) && value >= 0);
|
|
130
|
+
const overdue = rows.filter((entry) => entry.status === 'pending' && Date.parse(entry.expiresAt) <= now.getTime());
|
|
131
|
+
return {
|
|
132
|
+
generatedAt: now.toISOString(),
|
|
133
|
+
sampleSize: rows.length,
|
|
134
|
+
evidenceStatus: rows.length ? 'measured' : 'insufficient_evidence',
|
|
135
|
+
pending: rows.filter((entry) => entry.status === 'pending').length,
|
|
136
|
+
approved: rows.filter((entry) => entry.status === 'approved').length,
|
|
137
|
+
rejected: rows.filter((entry) => entry.status === 'rejected').length,
|
|
138
|
+
expired: rows.filter((entry) => entry.status === 'expired').length,
|
|
139
|
+
overdue: overdue.length,
|
|
140
|
+
medianDecisionLatencyMs: percentile(decisionLatencies, 0.5),
|
|
141
|
+
p95DecisionLatencyMs: percentile(decisionLatencies, 0.95),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function readEvents(options = {}) {
|
|
146
|
+
const inputPath = options.inputPath ? path.resolve(options.inputPath) : getEscalationsPath(options);
|
|
147
|
+
let raw = '';
|
|
148
|
+
try {
|
|
149
|
+
raw = fs.readFileSync(inputPath, 'utf8');
|
|
150
|
+
} catch {
|
|
151
|
+
return [];
|
|
152
|
+
}
|
|
153
|
+
return raw.split('\n').map((line) => line.trim()).filter(Boolean).flatMap((line) => {
|
|
154
|
+
try {
|
|
155
|
+
return [JSON.parse(line)];
|
|
156
|
+
} catch {
|
|
157
|
+
return [];
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function appendEvent(event, options) {
|
|
163
|
+
const outputPath = getEscalationsPath(options);
|
|
164
|
+
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
165
|
+
fs.appendFileSync(outputPath, `${JSON.stringify(event)}\n`, 'utf8');
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function requiredIdentity(value, field) {
|
|
169
|
+
if (!value || typeof value !== 'object') throw escalationError(`${field} identity is required`);
|
|
170
|
+
const identity = {
|
|
171
|
+
id: requiredString(value.id, `${field}.id`),
|
|
172
|
+
kind: requiredString(value.kind, `${field}.kind`),
|
|
173
|
+
};
|
|
174
|
+
const displayName = optionalString(value.displayName);
|
|
175
|
+
if (displayName) identity.displayName = displayName;
|
|
176
|
+
return identity;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function requiredString(value, field) {
|
|
180
|
+
const clean = String(value ?? '').trim();
|
|
181
|
+
if (!clean) throw escalationError(`${field} is required`);
|
|
182
|
+
return clean;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function optionalString(value) {
|
|
186
|
+
const clean = String(value ?? '').trim();
|
|
187
|
+
return clean || undefined;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function stringArray(value) {
|
|
191
|
+
return Array.isArray(value) ? value.map((entry) => String(entry).trim()).filter(Boolean) : [];
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function finiteNumber(value, fallback) {
|
|
195
|
+
const number = Number(value);
|
|
196
|
+
return Number.isFinite(number) ? number : fallback;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function sameIdentity(a, b) {
|
|
200
|
+
return a?.kind === b?.kind && a?.id === b?.id;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function eventHash(event) {
|
|
204
|
+
return crypto.createHash('sha256').update(stableStringify(event)).digest('hex');
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function eventComparableHash(event) {
|
|
208
|
+
const comparable = {
|
|
209
|
+
idempotencyKey: event.idempotencyKey,
|
|
210
|
+
taskId: event.taskId,
|
|
211
|
+
reason: event.reason,
|
|
212
|
+
severity: event.severity,
|
|
213
|
+
requester: event.requester,
|
|
214
|
+
evidence: event.evidence,
|
|
215
|
+
};
|
|
216
|
+
return crypto.createHash('sha256').update(stableStringify(comparable)).digest('hex');
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function stableStringify(value) {
|
|
220
|
+
if (!value || typeof value !== 'object') return JSON.stringify(value);
|
|
221
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
|
|
222
|
+
const keys = Object.keys(value).sort((left, right) => left.localeCompare(right));
|
|
223
|
+
const properties = keys.map((key) => [
|
|
224
|
+
JSON.stringify(key),
|
|
225
|
+
stableStringify(value[key]),
|
|
226
|
+
].join(':'));
|
|
227
|
+
return ['{', properties.join(','), '}'].join('');
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function percentile(values, quantile) {
|
|
231
|
+
if (!values.length) return null;
|
|
232
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
233
|
+
const index = Math.max(0, Math.ceil(sorted.length * quantile) - 1);
|
|
234
|
+
return sorted[index];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function escalationError(message) {
|
|
238
|
+
const error = new Error(`Invalid human escalation: ${message}`);
|
|
239
|
+
error.code = 'THUMBGATE_ESCALATION_INVALID';
|
|
240
|
+
return error;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function isCliInvocation() {
|
|
244
|
+
return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (isCliInvocation()) {
|
|
248
|
+
const command = process.argv[2] || 'metrics';
|
|
249
|
+
const escalations = listEscalations();
|
|
250
|
+
if (command === 'list') console.log(JSON.stringify(escalations, null, 2));
|
|
251
|
+
else if (command === 'metrics') console.log(JSON.stringify(calculateEscalationMetrics(escalations), null, 2));
|
|
252
|
+
else {
|
|
253
|
+
console.error('Usage: human-escalation.js [list|metrics]');
|
|
254
|
+
process.exitCode = 1;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
module.exports = {
|
|
259
|
+
calculateEscalationMetrics,
|
|
260
|
+
decideEscalation,
|
|
261
|
+
getEscalation,
|
|
262
|
+
getEscalationsPath,
|
|
263
|
+
listEscalations,
|
|
264
|
+
requestEscalation,
|
|
265
|
+
};
|
|
@@ -151,13 +151,14 @@ function isHookPromptEnvelope(context) {
|
|
|
151
151
|
parsed.transcriptPath
|
|
152
152
|
)
|
|
153
153
|
);
|
|
154
|
-
} catch
|
|
154
|
+
} catch {
|
|
155
|
+
// Not JSON — by definition not a hook envelope.
|
|
155
156
|
return false;
|
|
156
157
|
}
|
|
157
158
|
}
|
|
158
159
|
|
|
159
160
|
function patternContext(entry) {
|
|
160
|
-
const context = entry
|
|
161
|
+
const context = entry?.context ? String(entry.context) : '';
|
|
161
162
|
if (!context) return '';
|
|
162
163
|
const hasExplicitFeedback = Boolean(
|
|
163
164
|
entry.whatWentWrong ||
|
|
@@ -189,48 +190,6 @@ function isAutomatedFeedback(entry) {
|
|
|
189
190
|
}
|
|
190
191
|
|
|
191
192
|
|
|
192
|
-
function isHookPromptEnvelope(context) {
|
|
193
|
-
if (!context || typeof context !== 'string') return false;
|
|
194
|
-
try {
|
|
195
|
-
const parsed = JSON.parse(context);
|
|
196
|
-
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return false;
|
|
197
|
-
return Boolean(
|
|
198
|
-
parsed.prompt &&
|
|
199
|
-
(
|
|
200
|
-
parsed.hookEventName ||
|
|
201
|
-
parsed.hook_event_name ||
|
|
202
|
-
parsed.workspaceRoot ||
|
|
203
|
-
parsed.workspace_root ||
|
|
204
|
-
parsed.session_id ||
|
|
205
|
-
parsed.sessionId ||
|
|
206
|
-
parsed.transcript_path ||
|
|
207
|
-
parsed.transcriptPath
|
|
208
|
-
)
|
|
209
|
-
);
|
|
210
|
-
} catch (_) {
|
|
211
|
-
return false;
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
function patternContext(entry) {
|
|
216
|
-
const context = entry && entry.context ? String(entry.context) : '';
|
|
217
|
-
if (!context) return '';
|
|
218
|
-
const hasExplicitFeedback = Boolean(
|
|
219
|
-
entry.whatWentWrong ||
|
|
220
|
-
entry.what_went_wrong ||
|
|
221
|
-
entry.whatToChange ||
|
|
222
|
-
entry.what_to_change ||
|
|
223
|
-
entry.failureType ||
|
|
224
|
-
(Array.isArray(entry.tags) && entry.tags.length > 0) ||
|
|
225
|
-
entry.structuredRule
|
|
226
|
-
);
|
|
227
|
-
if (isHookPromptEnvelope(context) && !hasExplicitFeedback) return '';
|
|
228
|
-
if (isHookPromptEnvelope(context) && hasExplicitFeedback) {
|
|
229
|
-
return '';
|
|
230
|
-
}
|
|
231
|
-
return context;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
193
|
/**
|
|
235
194
|
* Extract ms from a timestamp value. Returns 0 on failure.
|
|
236
195
|
*/
|
|
@@ -494,14 +453,94 @@ function buildAdditionalContext(state, constraints, maxChars) {
|
|
|
494
453
|
* @param {string[]} words - keyword list from a pattern
|
|
495
454
|
* @returns {boolean}
|
|
496
455
|
*/
|
|
456
|
+
// Callers hand us the pending action in several shapes: a plain command string, an object,
|
|
457
|
+
// or a JSON envelope like {"toolName":…,"command":…,"filePath":…,"affectedFiles":[…]}.
|
|
458
|
+
// Matching over the raw JSON meant the envelope's own KEY NAMES were part of the haystack,
|
|
459
|
+
// so the tokens "files", "command", "tool", "name" and "path" were present on every single
|
|
460
|
+
// evaluation. With a two-hit block threshold, any guard whose keywords included two such
|
|
461
|
+
// common words blocked every action regardless of what that action was. Match on the VALUES
|
|
462
|
+
// only — the guard should key on the action, never on how we happened to serialize it.
|
|
463
|
+
// normalize() runs text through sanitizeFeedbackText(), which exists to reject hook
|
|
464
|
+
// TRANSPORT PAYLOADS and path-dominated blobs from human FEEDBACK. That is the wrong filter
|
|
465
|
+
// for the pending action we are matching against: a real action is frequently just a command
|
|
466
|
+
// plus a list of file paths, which sanitizeFeedbackText() discards wholesale as a "path blob",
|
|
467
|
+
// leaving an empty haystack and silently matching nothing. Apply only the redactions here.
|
|
468
|
+
function normalizeActionText(text) {
|
|
469
|
+
if (!text || typeof text !== 'string') return '';
|
|
470
|
+
return text
|
|
471
|
+
.replace(/\/Users\/[^\s/]+/g, '/Users/redacted')
|
|
472
|
+
.replace(/:\d{4,5}\b/g, ':PORT')
|
|
473
|
+
.toLowerCase()
|
|
474
|
+
.trim();
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function buildMatchHaystack(input) {
|
|
478
|
+
if (input == null) return '';
|
|
479
|
+
let value = input;
|
|
480
|
+
if (typeof value === 'string') {
|
|
481
|
+
const trimmed = value.trim();
|
|
482
|
+
if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return value;
|
|
483
|
+
try {
|
|
484
|
+
value = JSON.parse(trimmed);
|
|
485
|
+
} catch {
|
|
486
|
+
return value;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
if (typeof value !== 'object') return String(value);
|
|
490
|
+
|
|
491
|
+
const parts = [];
|
|
492
|
+
const seen = new Set();
|
|
493
|
+
const walk = (node, depth) => {
|
|
494
|
+
if (node == null || depth > 6) return;
|
|
495
|
+
if (typeof node === 'object') {
|
|
496
|
+
if (seen.has(node)) return;
|
|
497
|
+
seen.add(node);
|
|
498
|
+
for (const child of Array.isArray(node) ? node : Object.values(node)) walk(child, depth + 1);
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
if (typeof node === 'boolean') return; // "true"/"false" are structure, not content
|
|
502
|
+
parts.push(String(node));
|
|
503
|
+
};
|
|
504
|
+
walk(value, 0);
|
|
505
|
+
return parts.join(' ');
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// A guard word is "specific" when it is a compound identifier — keywords() preserves `-` and
|
|
509
|
+
// `_`, so a token like "generated-cache" or "tool_registry" survives intact and is almost
|
|
510
|
+
// always lifted from a real command, path or symbol rather than from prose. One such token
|
|
511
|
+
// is strong evidence on its own.
|
|
512
|
+
//
|
|
513
|
+
// Deliberately NOT keyed on length: ordinary English words ("deployment", "permission",
|
|
514
|
+
// "everything") are long but common, and letting one of them carry a block on its own would
|
|
515
|
+
// over-block. Those still require a second corroborating hit.
|
|
516
|
+
function isSpecificKeyword(word) {
|
|
517
|
+
return /[-_]/.test(word);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// Whole-word matching: a bare includes() let "app" hit "apps/", "application" and "happen".
|
|
521
|
+
// Boundaries are non-alphanumerics, so path and punctuation separators still delimit tokens
|
|
522
|
+
// (`src/jobs/queue.js` matches the word "jobs").
|
|
523
|
+
function containsWholeWord(haystack, word) {
|
|
524
|
+
const escaped = String(word).replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
|
|
525
|
+
try {
|
|
526
|
+
return new RegExp(`(?:^|[^a-z0-9])${escaped}(?:[^a-z0-9]|$)`, 'i').test(haystack);
|
|
527
|
+
} catch {
|
|
528
|
+
return haystack.includes(word);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
497
532
|
function hasTwoKeywordHits(normalizedInput, words) {
|
|
498
533
|
if (!normalizedInput || !words || words.length === 0) return false;
|
|
499
534
|
let hits = 0;
|
|
535
|
+
const seen = new Set();
|
|
500
536
|
for (const word of words) {
|
|
501
|
-
if (
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
537
|
+
if (!word || seen.has(word)) continue;
|
|
538
|
+
seen.add(word);
|
|
539
|
+
if (!containsWholeWord(normalizedInput, word)) continue;
|
|
540
|
+
// A specific compound/long token carries a match on its own; generic words need two.
|
|
541
|
+
if (isSpecificKeyword(word)) return true;
|
|
542
|
+
hits++;
|
|
543
|
+
if (hits >= 2) return true;
|
|
505
544
|
}
|
|
506
545
|
return false;
|
|
507
546
|
}
|
|
@@ -614,7 +653,7 @@ function evaluateCompiledGuards(artifact, toolName, toolInput) {
|
|
|
614
653
|
return { mode: 'allow', reason: '', source: 'compiled' };
|
|
615
654
|
}
|
|
616
655
|
|
|
617
|
-
const normInput =
|
|
656
|
+
const normInput = normalizeActionText(buildMatchHaystack(toolInput));
|
|
618
657
|
const normTool = (toolName || '').toLowerCase();
|
|
619
658
|
|
|
620
659
|
for (const guard of artifact.guards) {
|
|
@@ -656,7 +695,7 @@ function evaluateCompiledGuards(artifact, toolName, toolInput) {
|
|
|
656
695
|
* @returns {{ mode: string, reason: string, source: string }}
|
|
657
696
|
*/
|
|
658
697
|
function evaluatePretoolFromState(state, toolName, toolInput) {
|
|
659
|
-
const normInput =
|
|
698
|
+
const normInput = normalizeActionText(buildMatchHaystack(toolInput));
|
|
660
699
|
const normTool = (toolName || '').toLowerCase();
|
|
661
700
|
|
|
662
701
|
for (const pattern of state.recurringNegativePatterns || []) {
|
|
@@ -824,6 +863,10 @@ module.exports = {
|
|
|
824
863
|
keywords,
|
|
825
864
|
hashText,
|
|
826
865
|
hasTwoKeywordHits,
|
|
866
|
+
buildMatchHaystack,
|
|
867
|
+
normalizeActionText,
|
|
868
|
+
isSpecificKeyword,
|
|
869
|
+
containsWholeWord,
|
|
827
870
|
readJsonl,
|
|
828
871
|
getHybridPaths,
|
|
829
872
|
PATHS,
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Read recent JSONL rows since `sinceMs`, scanning at most maxBytes from EOF.
|
|
7
|
+
* Avoids loading multi-MB ledgers entirely into memory (export timeout fix).
|
|
8
|
+
*/
|
|
9
|
+
function readJsonlSinceTail(filePath, {
|
|
10
|
+
sinceMs = 0,
|
|
11
|
+
limit = 1000,
|
|
12
|
+
maxBytes = 8 * 1024 * 1024,
|
|
13
|
+
timestampKeys = ['timestamp', 'receivedAt', 'ts', 'createdAt'],
|
|
14
|
+
} = {}) {
|
|
15
|
+
if (!filePath || !fs.existsSync(filePath)) {
|
|
16
|
+
return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
let stat;
|
|
20
|
+
try {
|
|
21
|
+
stat = fs.statSync(filePath);
|
|
22
|
+
} catch {
|
|
23
|
+
return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const size = stat.size || 0;
|
|
27
|
+
if (size === 0) {
|
|
28
|
+
return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const scanBytes = Math.min(size, Math.max(64 * 1024, maxBytes));
|
|
32
|
+
const start = Math.max(0, size - scanBytes);
|
|
33
|
+
let buf;
|
|
34
|
+
try {
|
|
35
|
+
const fd = fs.openSync(filePath, 'r');
|
|
36
|
+
try {
|
|
37
|
+
buf = Buffer.alloc(scanBytes);
|
|
38
|
+
fs.readSync(fd, buf, 0, scanBytes, start);
|
|
39
|
+
} finally {
|
|
40
|
+
fs.closeSync(fd);
|
|
41
|
+
}
|
|
42
|
+
} catch {
|
|
43
|
+
return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let text = buf.toString('utf8');
|
|
47
|
+
// If we started mid-file, drop the partial first line.
|
|
48
|
+
if (start > 0) {
|
|
49
|
+
const nl = text.indexOf('\n');
|
|
50
|
+
text = nl >= 0 ? text.slice(nl + 1) : '';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const matched = [];
|
|
54
|
+
for (const line of text.split('\n')) {
|
|
55
|
+
if (!line) continue;
|
|
56
|
+
let obj;
|
|
57
|
+
try {
|
|
58
|
+
obj = JSON.parse(line);
|
|
59
|
+
} catch {
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
let parsed = NaN;
|
|
63
|
+
for (const key of timestampKeys) {
|
|
64
|
+
const raw = obj && obj[key];
|
|
65
|
+
if (raw == null) continue;
|
|
66
|
+
parsed = typeof raw === 'number' ? raw : Date.parse(raw);
|
|
67
|
+
if (Number.isFinite(parsed)) break;
|
|
68
|
+
}
|
|
69
|
+
// Preserve since filter: only include rows with a parseable timestamp
|
|
70
|
+
// inside the window. Legacy/malformed lines without timestamps must not
|
|
71
|
+
// inflate totalAfterSince or journey summaries.
|
|
72
|
+
if (Number.isFinite(parsed) && parsed >= sinceMs) {
|
|
73
|
+
matched.push(obj);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const totalAfterSince = matched.length;
|
|
78
|
+
const rows = matched.slice(-Math.max(1, limit));
|
|
79
|
+
return {
|
|
80
|
+
rows,
|
|
81
|
+
totalAfterSince,
|
|
82
|
+
truncated: totalAfterSince > rows.length || start > 0,
|
|
83
|
+
scannedBytes: scanBytes,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
module.exports = {
|
|
88
|
+
readJsonlSinceTail,
|
|
89
|
+
};
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
const fs = require('node:fs');
|
|
14
14
|
const path = require('node:path');
|
|
15
|
+
const { validateStructuredOutput } = require('./tool-contract-validator');
|
|
15
16
|
|
|
16
17
|
const DEFAULT_CRITERIA = [
|
|
17
18
|
{
|
|
@@ -117,15 +118,19 @@ function buildCompositeReward(sample = {}, options = {}) {
|
|
|
117
118
|
}
|
|
118
119
|
|
|
119
120
|
const judge = runJudgeSafely(sample, options.judge);
|
|
120
|
-
const
|
|
121
|
-
|
|
121
|
+
const score = judge.ok
|
|
122
|
+
? round((deterministic.score * 0.65) + (judge.score * 0.35))
|
|
123
|
+
: deterministic.score;
|
|
122
124
|
return {
|
|
123
125
|
score,
|
|
124
|
-
label: rewardLabel(score),
|
|
126
|
+
label: judge.ok ? rewardLabel(score) : 'deterministic_only',
|
|
125
127
|
deterministic,
|
|
126
128
|
judge,
|
|
127
|
-
|
|
128
|
-
|
|
129
|
+
scoringMode: judge.ok ? 'deterministic_plus_llm_judge' : 'deterministic_only',
|
|
130
|
+
failureMode: judge.ok ? [] : [judge.available ? 'judge_error' : 'judge_unavailable'],
|
|
131
|
+
recommendation: judge.ok
|
|
132
|
+
? rewardRecommendation(score)
|
|
133
|
+
: 'Use deterministic results only; do not represent unavailable judge evidence as a neutral judgment.',
|
|
129
134
|
};
|
|
130
135
|
}
|
|
131
136
|
|
|
@@ -148,7 +153,8 @@ function buildPreferenceJudgment(a, b, options = {}) {
|
|
|
148
153
|
function buildJudgeReadinessReport(samples = [], options = {}) {
|
|
149
154
|
const rewards = samples.map((sample) => buildCompositeReward(sample, options));
|
|
150
155
|
const blocked = rewards.filter((reward) => reward.label === 'deterministic_block');
|
|
151
|
-
const neutralFallbacks = rewards.filter((reward) =>
|
|
156
|
+
const neutralFallbacks = rewards.filter((reward) =>
|
|
157
|
+
reward.failureMode.includes('judge_error') || reward.failureMode.includes('judge_unavailable'));
|
|
152
158
|
return {
|
|
153
159
|
generatedAt: new Date().toISOString(),
|
|
154
160
|
samples: samples.length,
|
|
@@ -186,7 +192,7 @@ function measureJudgeConsistency(samples = [], judge = null, options = {}) {
|
|
|
186
192
|
|
|
187
193
|
function evaluateCriterion(id, prediction, sample, { requiresJson }) {
|
|
188
194
|
if (id === 'schema_valid') {
|
|
189
|
-
return evaluateSchemaCriterion(prediction, requiresJson);
|
|
195
|
+
return evaluateSchemaCriterion(prediction, requiresJson, sample.outputSchema);
|
|
190
196
|
}
|
|
191
197
|
if (id === 'grounded_evidence') {
|
|
192
198
|
return hasGroundedEvidence(prediction);
|
|
@@ -203,14 +209,17 @@ function evaluateCriterion(id, prediction, sample, { requiresJson }) {
|
|
|
203
209
|
return true;
|
|
204
210
|
}
|
|
205
211
|
|
|
206
|
-
function evaluateSchemaCriterion(prediction, requiresJson) {
|
|
212
|
+
function evaluateSchemaCriterion(prediction, requiresJson, outputSchema) {
|
|
207
213
|
if (!requiresJson) return true;
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
214
|
+
if (!outputSchema) {
|
|
215
|
+
try {
|
|
216
|
+
JSON.parse(prediction);
|
|
217
|
+
return true;
|
|
218
|
+
} catch {
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
213
221
|
}
|
|
222
|
+
return validateStructuredOutput(prediction, outputSchema).valid;
|
|
214
223
|
}
|
|
215
224
|
|
|
216
225
|
function hasGroundedEvidence(prediction) {
|
|
@@ -253,9 +262,10 @@ function buildCriterionReason(id, pass) {
|
|
|
253
262
|
function runJudgeSafely(sample, judge) {
|
|
254
263
|
if (typeof judge !== 'function') {
|
|
255
264
|
return {
|
|
256
|
-
ok:
|
|
257
|
-
|
|
258
|
-
|
|
265
|
+
ok: false,
|
|
266
|
+
available: false,
|
|
267
|
+
score: null,
|
|
268
|
+
rationale: 'No external judge configured; only deterministic checks are evidence.',
|
|
259
269
|
raw: null,
|
|
260
270
|
};
|
|
261
271
|
}
|
|
@@ -264,6 +274,7 @@ function runJudgeSafely(sample, judge) {
|
|
|
264
274
|
const score = clamp(Number(result.score ?? result), 0, 1);
|
|
265
275
|
return {
|
|
266
276
|
ok: true,
|
|
277
|
+
available: true,
|
|
267
278
|
score,
|
|
268
279
|
rationale: result.rationale || 'Judge returned a bounded score.',
|
|
269
280
|
raw: result,
|
|
@@ -271,8 +282,9 @@ function runJudgeSafely(sample, judge) {
|
|
|
271
282
|
} catch (err) {
|
|
272
283
|
return {
|
|
273
284
|
ok: false,
|
|
274
|
-
|
|
275
|
-
|
|
285
|
+
available: true,
|
|
286
|
+
score: null,
|
|
287
|
+
rationale: `Judge failed; deterministic checks remain authoritative. ${err.message}`,
|
|
276
288
|
raw: null,
|
|
277
289
|
};
|
|
278
290
|
}
|
|
@@ -99,16 +99,12 @@ function isEmbedderAvailable() {
|
|
|
99
99
|
const cfg = resolveGeminiEmbeddingConfig();
|
|
100
100
|
if (cfg && cfg.enabled && cfg.apiKey) return true;
|
|
101
101
|
} catch { /* policy module unavailable */ }
|
|
102
|
-
//
|
|
103
|
-
|
|
104
|
-
require.resolve('@huggingface/transformers');
|
|
105
|
-
return true;
|
|
106
|
-
} catch { /* not installed */ }
|
|
107
|
-
return false;
|
|
102
|
+
// The zero-dependency feature-hash provider is always available locally.
|
|
103
|
+
return true;
|
|
108
104
|
}
|
|
109
105
|
|
|
110
106
|
function defaultEmbedder() {
|
|
111
|
-
// Lazy: do not pull in LanceDB
|
|
107
|
+
// Lazy: do not pull in LanceDB or optional embedding providers at module require time.
|
|
112
108
|
const { embed } = require('./vector-store');
|
|
113
109
|
return embed;
|
|
114
110
|
}
|