thumbgate 1.28.4 → 1.29.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/.claude/commands/dashboard.md +11 -1
  2. package/.claude/commands/thumbgate-dashboard.md +23 -8
  3. package/.claude-plugin/plugin.json +1 -1
  4. package/.well-known/llms.txt +18 -10
  5. package/.well-known/mcp/server-card.json +1 -1
  6. package/README.md +66 -3
  7. package/adapters/claude/.mcp.json +2 -2
  8. package/adapters/forge/forge.yaml +3 -3
  9. package/adapters/mcp/server-stdio.js +88 -2
  10. package/adapters/opencode/opencode.json +1 -1
  11. package/bin/cli.js +8 -8
  12. package/bin/postinstall.js +4 -13
  13. package/commands/dashboard.md +11 -1
  14. package/commands/thumbgate-dashboard.md +23 -8
  15. package/config/agent-outcome-monitor-thresholds.json +63 -0
  16. package/config/evals/agent-outcomes-baseline.json +17 -0
  17. package/config/evals/agent-outcomes-golden.json +412 -0
  18. package/config/evals/prompt-eval-baseline.json +23 -0
  19. package/config/github-about.json +5 -4
  20. package/config/post-deploy-marketing-pages.json +6 -6
  21. package/config/schemas/task-outcome-receipt.schema.json +296 -0
  22. package/docs/integrations/grafana/README.md +109 -0
  23. package/docs/integrations/grafana/thumbgate-revenue-evidence-dashboard.json +1930 -0
  24. package/openapi/openapi.yaml +475 -5
  25. package/package.json +75 -22
  26. package/public/agent-manager.html +10 -11
  27. package/public/agents-cost-savings.html +2 -2
  28. package/public/assets/brand/thumbgate-logo-transparent.svg +6 -11
  29. package/public/assets/brand/thumbgate-mark-inline-v3.svg +11 -10
  30. package/public/assets/brand/thumbgate-mark.svg +10 -11
  31. package/public/blog/inside-your-boundary.html +114 -0
  32. package/public/blog/process-over-outcome-gates.html +119 -0
  33. package/public/blog.html +296 -402
  34. package/public/brand/thumbgate-mark.svg +5 -9
  35. package/public/codex-enterprise.html +2 -2
  36. package/public/compare.html +12 -3
  37. package/public/diagnostic.html +79 -29
  38. package/public/guide.html +4 -4
  39. package/public/index.html +1090 -2098
  40. package/public/install.html +3 -3
  41. package/public/js/buyer-intent.js +33 -18
  42. package/public/numbers.html +2 -2
  43. package/public/pricing.html +268 -408
  44. package/public/pro.html +4 -4
  45. package/scripts/agent-outcome-eval.js +130 -0
  46. package/scripts/agent-outcome-monitor.js +261 -0
  47. package/scripts/agent-reasoning-traces.js +8 -9
  48. package/scripts/async-job-runner.js +107 -13
  49. package/scripts/billing.js +456 -126
  50. package/scripts/buyer-paths.js +102 -0
  51. package/scripts/cli-feedback.js +2 -2
  52. package/scripts/commercial-offer.js +18 -10
  53. package/scripts/durability/step.js +121 -12
  54. package/scripts/external-customer-audit.js +881 -0
  55. package/scripts/feedback-loop.js +26 -0
  56. package/scripts/gates-engine.js +554 -19
  57. package/scripts/grafana-revenue-evidence.js +856 -0
  58. package/scripts/human-escalation.js +265 -0
  59. package/scripts/hybrid-feedback-context.js +93 -50
  60. package/scripts/jsonl-window.js +89 -0
  61. package/scripts/judge-reward-function.js +30 -18
  62. package/scripts/lesson-embedding-index.js +3 -7
  63. package/scripts/meta-agent-loop.js +20 -2
  64. package/scripts/observability-env.js +139 -0
  65. package/scripts/observability-setup.js +55 -0
  66. package/scripts/plausible-domain-config.js +4 -0
  67. package/scripts/prompt-eval.js +81 -4
  68. package/scripts/provider-live-evidence.js +1290 -0
  69. package/scripts/provider-payment-reconciler.js +442 -0
  70. package/scripts/provider-revenue-evidence.js +249 -0
  71. package/scripts/rate-limiter.js +1 -5
  72. package/scripts/revenue-action-eligibility.js +414 -0
  73. package/scripts/revenue-evidence-remediation.js +694 -0
  74. package/scripts/revenue-offer-system.js +709 -0
  75. package/scripts/sales-pipeline.js +1117 -0
  76. package/scripts/schedule-manager.js +249 -0
  77. package/scripts/seo-gsd.js +8 -4
  78. package/scripts/stripe-credentials.js +37 -0
  79. package/scripts/stripe-revenue-catalog-audit.js +363 -0
  80. package/scripts/stripe-revenue-catalog.js +164 -0
  81. package/scripts/task-outcomes.js +425 -0
  82. package/scripts/telemetry-analytics.js +23 -3
  83. package/scripts/tool-contract-validator.js +287 -59
  84. package/scripts/tool-registry.js +143 -0
  85. package/scripts/vector-store.js +83 -7
  86. package/scripts/workflow-intake-queue.js +483 -0
  87. package/src/api/server.js +647 -118
@@ -0,0 +1,265 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ /**
5
+ * Human Escalation Queue
6
+ *
7
+ * Agents may request or inspect escalation, but approval decisions require an
8
+ * explicit human actor identity distinct from the requesting agent. Events are
9
+ * append-only so every transition remains auditable.
10
+ */
11
+
12
+ const crypto = require('node:crypto');
13
+ const fs = require('node:fs');
14
+ const path = require('node:path');
15
+ const { getFeedbackPaths } = require('./feedback-paths');
16
+
17
+ const ESCALATIONS_FILE = 'human-escalations.jsonl';
18
+ const MAX_TTL_MS = 7 * 24 * 60 * 60 * 1000;
19
+ const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
20
+ const SEVERITIES = new Set(['low', 'medium', 'high', 'critical']);
21
+ const DECISIONS = new Set(['approved', 'rejected', 'cancelled']);
22
+
23
+ function getEscalationsPath(options = {}) {
24
+ return path.join(getFeedbackPaths(options).FEEDBACK_DIR, ESCALATIONS_FILE);
25
+ }
26
+
27
+ function requestEscalation(input = {}, options = {}) {
28
+ const now = options.now || new Date();
29
+ const taskId = requiredString(input.taskId, 'taskId');
30
+ const reason = requiredString(input.reason, 'reason');
31
+ const requester = requiredIdentity(input.requester, 'requester');
32
+ const evidence = stringArray(input.evidence);
33
+ if (evidence.length === 0) throw escalationError('evidence must contain at least one item');
34
+ const severity = input.severity || 'medium';
35
+ if (!SEVERITIES.has(severity)) throw escalationError(`severity must be one of ${Array.from(SEVERITIES).join(', ')}`);
36
+ const ttlMs = Math.min(MAX_TTL_MS, Math.max(1, finiteNumber(input.ttlMs, DEFAULT_TTL_MS)));
37
+ const idempotencyKey = requiredString(input.idempotencyKey || taskId, 'idempotencyKey');
38
+ const existing = listEscalations(options).find((entry) => entry.idempotencyKey === idempotencyKey);
39
+
40
+ const request = {
41
+ escalationId: input.escalationId || `esc_${crypto.randomUUID()}`,
42
+ idempotencyKey,
43
+ taskId,
44
+ reason,
45
+ severity,
46
+ requester,
47
+ evidence,
48
+ requestedAt: now.toISOString(),
49
+ expiresAt: new Date(now.getTime() + ttlMs).toISOString(),
50
+ status: 'pending',
51
+ eventType: 'requested',
52
+ };
53
+ request.eventHash = eventHash(request);
54
+
55
+ if (existing) {
56
+ if (eventComparableHash(existing) !== eventComparableHash(request)) {
57
+ const error = escalationError(`conflicting request for idempotency key '${idempotencyKey}'`);
58
+ error.code = 'THUMBGATE_IDEMPOTENCY_CONFLICT';
59
+ throw error;
60
+ }
61
+ return { recorded: false, duplicate: true, escalation: existing };
62
+ }
63
+
64
+ appendEvent(request, options);
65
+ return { recorded: true, duplicate: false, escalation: request };
66
+ }
67
+
68
+ function decideEscalation(input = {}, options = {}) {
69
+ const escalationId = requiredString(input.escalationId, 'escalationId');
70
+ const decision = requiredString(input.decision, 'decision');
71
+ if (!DECISIONS.has(decision)) throw escalationError(`decision must be one of ${Array.from(DECISIONS).join(', ')}`);
72
+ if (Object.hasOwn(input, 'actor')) {
73
+ throw escalationError('decision actor is derived from the authenticated reviewer and must not be supplied by the caller');
74
+ }
75
+ const actor = requiredIdentity(options.authenticatedActor, 'authenticatedActor');
76
+ if (actor.kind !== 'human') throw escalationError('authenticatedActor.kind must be human');
77
+ const reason = requiredString(input.reason, 'reason');
78
+ const current = getEscalation(escalationId, options);
79
+ if (!current) throw escalationError(`unknown escalation '${escalationId}'`);
80
+ if (current.status !== 'pending') throw escalationError(`escalation '${escalationId}' is already ${current.status}`);
81
+ if (sameIdentity(current.requester, actor)) throw escalationError('requester cannot decide their own escalation');
82
+
83
+ const now = options.now || new Date();
84
+ const event = {
85
+ escalationId,
86
+ taskId: current.taskId,
87
+ status: decision,
88
+ eventType: 'decided',
89
+ decision,
90
+ actor,
91
+ reason,
92
+ decidedAt: now.toISOString(),
93
+ };
94
+ event.eventHash = eventHash(event);
95
+ appendEvent(event, options);
96
+ return { recorded: true, escalation: { ...current, ...event } };
97
+ }
98
+
99
+ function listEscalations(options = {}) {
100
+ const events = readEvents(options);
101
+ const byId = new Map();
102
+ for (const event of events) {
103
+ const current = byId.get(event.escalationId) || {};
104
+ byId.set(event.escalationId, { ...current, ...event });
105
+ }
106
+
107
+ const nowMs = (options.now || new Date()).getTime();
108
+ const rows = Array.from(byId.values()).map((entry) => {
109
+ if (entry.status === 'pending' && Date.parse(entry.expiresAt) <= nowMs) {
110
+ return { ...entry, status: 'expired' };
111
+ }
112
+ return entry;
113
+ });
114
+ const status = options.status;
115
+ return rows
116
+ .filter((entry) => !status || entry.status === status)
117
+ .sort((a, b) => Date.parse(b.requestedAt || b.decidedAt) - Date.parse(a.requestedAt || a.decidedAt));
118
+ }
119
+
120
+ function getEscalation(escalationId, options = {}) {
121
+ return listEscalations(options).find((entry) => entry.escalationId === escalationId) || null;
122
+ }
123
+
124
+ function calculateEscalationMetrics(escalations = [], now = new Date()) {
125
+ const rows = escalations.filter(Boolean);
126
+ const decided = rows.filter((entry) => ['approved', 'rejected'].includes(entry.status));
127
+ const decisionLatencies = decided
128
+ .map((entry) => Date.parse(entry.decidedAt) - Date.parse(entry.requestedAt))
129
+ .filter((value) => Number.isFinite(value) && value >= 0);
130
+ const overdue = rows.filter((entry) => entry.status === 'pending' && Date.parse(entry.expiresAt) <= now.getTime());
131
+ return {
132
+ generatedAt: now.toISOString(),
133
+ sampleSize: rows.length,
134
+ evidenceStatus: rows.length ? 'measured' : 'insufficient_evidence',
135
+ pending: rows.filter((entry) => entry.status === 'pending').length,
136
+ approved: rows.filter((entry) => entry.status === 'approved').length,
137
+ rejected: rows.filter((entry) => entry.status === 'rejected').length,
138
+ expired: rows.filter((entry) => entry.status === 'expired').length,
139
+ overdue: overdue.length,
140
+ medianDecisionLatencyMs: percentile(decisionLatencies, 0.5),
141
+ p95DecisionLatencyMs: percentile(decisionLatencies, 0.95),
142
+ };
143
+ }
144
+
145
+ function readEvents(options = {}) {
146
+ const inputPath = options.inputPath ? path.resolve(options.inputPath) : getEscalationsPath(options);
147
+ let raw = '';
148
+ try {
149
+ raw = fs.readFileSync(inputPath, 'utf8');
150
+ } catch {
151
+ return [];
152
+ }
153
+ return raw.split('\n').map((line) => line.trim()).filter(Boolean).flatMap((line) => {
154
+ try {
155
+ return [JSON.parse(line)];
156
+ } catch {
157
+ return [];
158
+ }
159
+ });
160
+ }
161
+
162
+ function appendEvent(event, options) {
163
+ const outputPath = getEscalationsPath(options);
164
+ fs.mkdirSync(path.dirname(outputPath), { recursive: true });
165
+ fs.appendFileSync(outputPath, `${JSON.stringify(event)}\n`, 'utf8');
166
+ }
167
+
168
+ function requiredIdentity(value, field) {
169
+ if (!value || typeof value !== 'object') throw escalationError(`${field} identity is required`);
170
+ const identity = {
171
+ id: requiredString(value.id, `${field}.id`),
172
+ kind: requiredString(value.kind, `${field}.kind`),
173
+ };
174
+ const displayName = optionalString(value.displayName);
175
+ if (displayName) identity.displayName = displayName;
176
+ return identity;
177
+ }
178
+
179
+ function requiredString(value, field) {
180
+ const clean = String(value ?? '').trim();
181
+ if (!clean) throw escalationError(`${field} is required`);
182
+ return clean;
183
+ }
184
+
185
+ function optionalString(value) {
186
+ const clean = String(value ?? '').trim();
187
+ return clean || undefined;
188
+ }
189
+
190
+ function stringArray(value) {
191
+ return Array.isArray(value) ? value.map((entry) => String(entry).trim()).filter(Boolean) : [];
192
+ }
193
+
194
+ function finiteNumber(value, fallback) {
195
+ const number = Number(value);
196
+ return Number.isFinite(number) ? number : fallback;
197
+ }
198
+
199
+ function sameIdentity(a, b) {
200
+ return a?.kind === b?.kind && a?.id === b?.id;
201
+ }
202
+
203
+ function eventHash(event) {
204
+ return crypto.createHash('sha256').update(stableStringify(event)).digest('hex');
205
+ }
206
+
207
+ function eventComparableHash(event) {
208
+ const comparable = {
209
+ idempotencyKey: event.idempotencyKey,
210
+ taskId: event.taskId,
211
+ reason: event.reason,
212
+ severity: event.severity,
213
+ requester: event.requester,
214
+ evidence: event.evidence,
215
+ };
216
+ return crypto.createHash('sha256').update(stableStringify(comparable)).digest('hex');
217
+ }
218
+
219
+ function stableStringify(value) {
220
+ if (!value || typeof value !== 'object') return JSON.stringify(value);
221
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
222
+ const keys = Object.keys(value).sort((left, right) => left.localeCompare(right));
223
+ const properties = keys.map((key) => [
224
+ JSON.stringify(key),
225
+ stableStringify(value[key]),
226
+ ].join(':'));
227
+ return ['{', properties.join(','), '}'].join('');
228
+ }
229
+
230
+ function percentile(values, quantile) {
231
+ if (!values.length) return null;
232
+ const sorted = [...values].sort((a, b) => a - b);
233
+ const index = Math.max(0, Math.ceil(sorted.length * quantile) - 1);
234
+ return sorted[index];
235
+ }
236
+
237
+ function escalationError(message) {
238
+ const error = new Error(`Invalid human escalation: ${message}`);
239
+ error.code = 'THUMBGATE_ESCALATION_INVALID';
240
+ return error;
241
+ }
242
+
243
+ function isCliInvocation() {
244
+ return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
245
+ }
246
+
247
+ if (isCliInvocation()) {
248
+ const command = process.argv[2] || 'metrics';
249
+ const escalations = listEscalations();
250
+ if (command === 'list') console.log(JSON.stringify(escalations, null, 2));
251
+ else if (command === 'metrics') console.log(JSON.stringify(calculateEscalationMetrics(escalations), null, 2));
252
+ else {
253
+ console.error('Usage: human-escalation.js [list|metrics]');
254
+ process.exitCode = 1;
255
+ }
256
+ }
257
+
258
+ module.exports = {
259
+ calculateEscalationMetrics,
260
+ decideEscalation,
261
+ getEscalation,
262
+ getEscalationsPath,
263
+ listEscalations,
264
+ requestEscalation,
265
+ };
@@ -151,13 +151,14 @@ function isHookPromptEnvelope(context) {
151
151
  parsed.transcriptPath
152
152
  )
153
153
  );
154
- } catch (_) {
154
+ } catch {
155
+ // Not JSON — by definition not a hook envelope.
155
156
  return false;
156
157
  }
157
158
  }
158
159
 
159
160
  function patternContext(entry) {
160
- const context = entry && entry.context ? String(entry.context) : '';
161
+ const context = entry?.context ? String(entry.context) : '';
161
162
  if (!context) return '';
162
163
  const hasExplicitFeedback = Boolean(
163
164
  entry.whatWentWrong ||
@@ -189,48 +190,6 @@ function isAutomatedFeedback(entry) {
189
190
  }
190
191
 
191
192
 
192
- function isHookPromptEnvelope(context) {
193
- if (!context || typeof context !== 'string') return false;
194
- try {
195
- const parsed = JSON.parse(context);
196
- if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return false;
197
- return Boolean(
198
- parsed.prompt &&
199
- (
200
- parsed.hookEventName ||
201
- parsed.hook_event_name ||
202
- parsed.workspaceRoot ||
203
- parsed.workspace_root ||
204
- parsed.session_id ||
205
- parsed.sessionId ||
206
- parsed.transcript_path ||
207
- parsed.transcriptPath
208
- )
209
- );
210
- } catch (_) {
211
- return false;
212
- }
213
- }
214
-
215
- function patternContext(entry) {
216
- const context = entry && entry.context ? String(entry.context) : '';
217
- if (!context) return '';
218
- const hasExplicitFeedback = Boolean(
219
- entry.whatWentWrong ||
220
- entry.what_went_wrong ||
221
- entry.whatToChange ||
222
- entry.what_to_change ||
223
- entry.failureType ||
224
- (Array.isArray(entry.tags) && entry.tags.length > 0) ||
225
- entry.structuredRule
226
- );
227
- if (isHookPromptEnvelope(context) && !hasExplicitFeedback) return '';
228
- if (isHookPromptEnvelope(context) && hasExplicitFeedback) {
229
- return '';
230
- }
231
- return context;
232
- }
233
-
234
193
  /**
235
194
  * Extract ms from a timestamp value. Returns 0 on failure.
236
195
  */
@@ -494,14 +453,94 @@ function buildAdditionalContext(state, constraints, maxChars) {
494
453
  * @param {string[]} words - keyword list from a pattern
495
454
  * @returns {boolean}
496
455
  */
456
+ // Callers hand us the pending action in several shapes: a plain command string, an object,
457
+ // or a JSON envelope like {"toolName":…,"command":…,"filePath":…,"affectedFiles":[…]}.
458
+ // Matching over the raw JSON meant the envelope's own KEY NAMES were part of the haystack,
459
+ // so the tokens "files", "command", "tool", "name" and "path" were present on every single
460
+ // evaluation. With a two-hit block threshold, any guard whose keywords included two such
461
+ // common words blocked every action regardless of what that action was. Match on the VALUES
462
+ // only — the guard should key on the action, never on how we happened to serialize it.
463
+ // normalize() runs text through sanitizeFeedbackText(), which exists to reject hook
464
+ // TRANSPORT PAYLOADS and path-dominated blobs from human FEEDBACK. That is the wrong filter
465
+ // for the pending action we are matching against: a real action is frequently just a command
466
+ // plus a list of file paths, which sanitizeFeedbackText() discards wholesale as a "path blob",
467
+ // leaving an empty haystack and silently matching nothing. Apply only the redactions here.
468
+ function normalizeActionText(text) {
469
+ if (!text || typeof text !== 'string') return '';
470
+ return text
471
+ .replace(/\/Users\/[^\s/]+/g, '/Users/redacted')
472
+ .replace(/:\d{4,5}\b/g, ':PORT')
473
+ .toLowerCase()
474
+ .trim();
475
+ }
476
+
477
+ function buildMatchHaystack(input) {
478
+ if (input == null) return '';
479
+ let value = input;
480
+ if (typeof value === 'string') {
481
+ const trimmed = value.trim();
482
+ if (!(trimmed.startsWith('{') || trimmed.startsWith('['))) return value;
483
+ try {
484
+ value = JSON.parse(trimmed);
485
+ } catch {
486
+ return value;
487
+ }
488
+ }
489
+ if (typeof value !== 'object') return String(value);
490
+
491
+ const parts = [];
492
+ const seen = new Set();
493
+ const walk = (node, depth) => {
494
+ if (node == null || depth > 6) return;
495
+ if (typeof node === 'object') {
496
+ if (seen.has(node)) return;
497
+ seen.add(node);
498
+ for (const child of Array.isArray(node) ? node : Object.values(node)) walk(child, depth + 1);
499
+ return;
500
+ }
501
+ if (typeof node === 'boolean') return; // "true"/"false" are structure, not content
502
+ parts.push(String(node));
503
+ };
504
+ walk(value, 0);
505
+ return parts.join(' ');
506
+ }
507
+
508
+ // A guard word is "specific" when it is a compound identifier — keywords() preserves `-` and
509
+ // `_`, so a token like "generated-cache" or "tool_registry" survives intact and is almost
510
+ // always lifted from a real command, path or symbol rather than from prose. One such token
511
+ // is strong evidence on its own.
512
+ //
513
+ // Deliberately NOT keyed on length: ordinary English words ("deployment", "permission",
514
+ // "everything") are long but common, and letting one of them carry a block on its own would
515
+ // over-block. Those still require a second corroborating hit.
516
+ function isSpecificKeyword(word) {
517
+ return /[-_]/.test(word);
518
+ }
519
+
520
+ // Whole-word matching: a bare includes() let "app" hit "apps/", "application" and "happen".
521
+ // Boundaries are non-alphanumerics, so path and punctuation separators still delimit tokens
522
+ // (`src/jobs/queue.js` matches the word "jobs").
523
+ function containsWholeWord(haystack, word) {
524
+ const escaped = String(word).replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
525
+ try {
526
+ return new RegExp(`(?:^|[^a-z0-9])${escaped}(?:[^a-z0-9]|$)`, 'i').test(haystack);
527
+ } catch {
528
+ return haystack.includes(word);
529
+ }
530
+ }
531
+
497
532
  function hasTwoKeywordHits(normalizedInput, words) {
498
533
  if (!normalizedInput || !words || words.length === 0) return false;
499
534
  let hits = 0;
535
+ const seen = new Set();
500
536
  for (const word of words) {
501
- if (normalizedInput.includes(word)) {
502
- hits++;
503
- if (hits >= 2) return true;
504
- }
537
+ if (!word || seen.has(word)) continue;
538
+ seen.add(word);
539
+ if (!containsWholeWord(normalizedInput, word)) continue;
540
+ // A specific compound/long token carries a match on its own; generic words need two.
541
+ if (isSpecificKeyword(word)) return true;
542
+ hits++;
543
+ if (hits >= 2) return true;
505
544
  }
506
545
  return false;
507
546
  }
@@ -614,7 +653,7 @@ function evaluateCompiledGuards(artifact, toolName, toolInput) {
614
653
  return { mode: 'allow', reason: '', source: 'compiled' };
615
654
  }
616
655
 
617
- const normInput = normalize(toolInput || '');
656
+ const normInput = normalizeActionText(buildMatchHaystack(toolInput));
618
657
  const normTool = (toolName || '').toLowerCase();
619
658
 
620
659
  for (const guard of artifact.guards) {
@@ -656,7 +695,7 @@ function evaluateCompiledGuards(artifact, toolName, toolInput) {
656
695
  * @returns {{ mode: string, reason: string, source: string }}
657
696
  */
658
697
  function evaluatePretoolFromState(state, toolName, toolInput) {
659
- const normInput = normalize(toolInput || '');
698
+ const normInput = normalizeActionText(buildMatchHaystack(toolInput));
660
699
  const normTool = (toolName || '').toLowerCase();
661
700
 
662
701
  for (const pattern of state.recurringNegativePatterns || []) {
@@ -824,6 +863,10 @@ module.exports = {
824
863
  keywords,
825
864
  hashText,
826
865
  hasTwoKeywordHits,
866
+ buildMatchHaystack,
867
+ normalizeActionText,
868
+ isSpecificKeyword,
869
+ containsWholeWord,
827
870
  readJsonl,
828
871
  getHybridPaths,
829
872
  PATHS,
@@ -0,0 +1,89 @@
1
+ 'use strict';
2
+
3
+ const fs = require('node:fs');
4
+
5
+ /**
6
+ * Read recent JSONL rows since `sinceMs`, scanning at most maxBytes from EOF.
7
+ * Avoids loading multi-MB ledgers entirely into memory (export timeout fix).
8
+ */
9
+ function readJsonlSinceTail(filePath, {
10
+ sinceMs = 0,
11
+ limit = 1000,
12
+ maxBytes = 8 * 1024 * 1024,
13
+ timestampKeys = ['timestamp', 'receivedAt', 'ts', 'createdAt'],
14
+ } = {}) {
15
+ if (!filePath || !fs.existsSync(filePath)) {
16
+ return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
17
+ }
18
+
19
+ let stat;
20
+ try {
21
+ stat = fs.statSync(filePath);
22
+ } catch {
23
+ return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
24
+ }
25
+
26
+ const size = stat.size || 0;
27
+ if (size === 0) {
28
+ return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
29
+ }
30
+
31
+ const scanBytes = Math.min(size, Math.max(64 * 1024, maxBytes));
32
+ const start = Math.max(0, size - scanBytes);
33
+ let buf;
34
+ try {
35
+ const fd = fs.openSync(filePath, 'r');
36
+ try {
37
+ buf = Buffer.alloc(scanBytes);
38
+ fs.readSync(fd, buf, 0, scanBytes, start);
39
+ } finally {
40
+ fs.closeSync(fd);
41
+ }
42
+ } catch {
43
+ return { rows: [], totalAfterSince: 0, truncated: false, scannedBytes: 0 };
44
+ }
45
+
46
+ let text = buf.toString('utf8');
47
+ // If we started mid-file, drop the partial first line.
48
+ if (start > 0) {
49
+ const nl = text.indexOf('\n');
50
+ text = nl >= 0 ? text.slice(nl + 1) : '';
51
+ }
52
+
53
+ const matched = [];
54
+ for (const line of text.split('\n')) {
55
+ if (!line) continue;
56
+ let obj;
57
+ try {
58
+ obj = JSON.parse(line);
59
+ } catch {
60
+ continue;
61
+ }
62
+ let parsed = NaN;
63
+ for (const key of timestampKeys) {
64
+ const raw = obj && obj[key];
65
+ if (raw == null) continue;
66
+ parsed = typeof raw === 'number' ? raw : Date.parse(raw);
67
+ if (Number.isFinite(parsed)) break;
68
+ }
69
+ // Preserve since filter: only include rows with a parseable timestamp
70
+ // inside the window. Legacy/malformed lines without timestamps must not
71
+ // inflate totalAfterSince or journey summaries.
72
+ if (Number.isFinite(parsed) && parsed >= sinceMs) {
73
+ matched.push(obj);
74
+ }
75
+ }
76
+
77
+ const totalAfterSince = matched.length;
78
+ const rows = matched.slice(-Math.max(1, limit));
79
+ return {
80
+ rows,
81
+ totalAfterSince,
82
+ truncated: totalAfterSince > rows.length || start > 0,
83
+ scannedBytes: scanBytes,
84
+ };
85
+ }
86
+
87
+ module.exports = {
88
+ readJsonlSinceTail,
89
+ };
@@ -12,6 +12,7 @@
12
12
 
13
13
  const fs = require('node:fs');
14
14
  const path = require('node:path');
15
+ const { validateStructuredOutput } = require('./tool-contract-validator');
15
16
 
16
17
  const DEFAULT_CRITERIA = [
17
18
  {
@@ -117,15 +118,19 @@ function buildCompositeReward(sample = {}, options = {}) {
117
118
  }
118
119
 
119
120
  const judge = runJudgeSafely(sample, options.judge);
120
- const judgeScore = judge.ok ? judge.score : 0.5;
121
- const score = round((deterministic.score * 0.65) + (judgeScore * 0.35));
121
+ const score = judge.ok
122
+ ? round((deterministic.score * 0.65) + (judge.score * 0.35))
123
+ : deterministic.score;
122
124
  return {
123
125
  score,
124
- label: rewardLabel(score),
126
+ label: judge.ok ? rewardLabel(score) : 'deterministic_only',
125
127
  deterministic,
126
128
  judge,
127
- failureMode: judge.ok ? [] : ['judge_error_neutral_reward'],
128
- recommendation: rewardRecommendation(score),
129
+ scoringMode: judge.ok ? 'deterministic_plus_llm_judge' : 'deterministic_only',
130
+ failureMode: judge.ok ? [] : [judge.available ? 'judge_error' : 'judge_unavailable'],
131
+ recommendation: judge.ok
132
+ ? rewardRecommendation(score)
133
+ : 'Use deterministic results only; do not represent unavailable judge evidence as a neutral judgment.',
129
134
  };
130
135
  }
131
136
 
@@ -148,7 +153,8 @@ function buildPreferenceJudgment(a, b, options = {}) {
148
153
  function buildJudgeReadinessReport(samples = [], options = {}) {
149
154
  const rewards = samples.map((sample) => buildCompositeReward(sample, options));
150
155
  const blocked = rewards.filter((reward) => reward.label === 'deterministic_block');
151
- const neutralFallbacks = rewards.filter((reward) => reward.failureMode.includes('judge_error_neutral_reward'));
156
+ const neutralFallbacks = rewards.filter((reward) =>
157
+ reward.failureMode.includes('judge_error') || reward.failureMode.includes('judge_unavailable'));
152
158
  return {
153
159
  generatedAt: new Date().toISOString(),
154
160
  samples: samples.length,
@@ -186,7 +192,7 @@ function measureJudgeConsistency(samples = [], judge = null, options = {}) {
186
192
 
187
193
  function evaluateCriterion(id, prediction, sample, { requiresJson }) {
188
194
  if (id === 'schema_valid') {
189
- return evaluateSchemaCriterion(prediction, requiresJson);
195
+ return evaluateSchemaCriterion(prediction, requiresJson, sample.outputSchema);
190
196
  }
191
197
  if (id === 'grounded_evidence') {
192
198
  return hasGroundedEvidence(prediction);
@@ -203,14 +209,17 @@ function evaluateCriterion(id, prediction, sample, { requiresJson }) {
203
209
  return true;
204
210
  }
205
211
 
206
- function evaluateSchemaCriterion(prediction, requiresJson) {
212
+ function evaluateSchemaCriterion(prediction, requiresJson, outputSchema) {
207
213
  if (!requiresJson) return true;
208
- try {
209
- JSON.parse(prediction);
210
- return true;
211
- } catch {
212
- return false;
214
+ if (!outputSchema) {
215
+ try {
216
+ JSON.parse(prediction);
217
+ return true;
218
+ } catch {
219
+ return false;
220
+ }
213
221
  }
222
+ return validateStructuredOutput(prediction, outputSchema).valid;
214
223
  }
215
224
 
216
225
  function hasGroundedEvidence(prediction) {
@@ -253,9 +262,10 @@ function buildCriterionReason(id, pass) {
253
262
  function runJudgeSafely(sample, judge) {
254
263
  if (typeof judge !== 'function') {
255
264
  return {
256
- ok: true,
257
- score: 0.5,
258
- rationale: 'No external judge configured; deterministic checks carried the reward.',
265
+ ok: false,
266
+ available: false,
267
+ score: null,
268
+ rationale: 'No external judge configured; only deterministic checks are evidence.',
259
269
  raw: null,
260
270
  };
261
271
  }
@@ -264,6 +274,7 @@ function runJudgeSafely(sample, judge) {
264
274
  const score = clamp(Number(result.score ?? result), 0, 1);
265
275
  return {
266
276
  ok: true,
277
+ available: true,
267
278
  score,
268
279
  rationale: result.rationale || 'Judge returned a bounded score.',
269
280
  raw: result,
@@ -271,8 +282,9 @@ function runJudgeSafely(sample, judge) {
271
282
  } catch (err) {
272
283
  return {
273
284
  ok: false,
274
- score: 0.5,
275
- rationale: `Judge failed; returned neutral reward. ${err.message}`,
285
+ available: true,
286
+ score: null,
287
+ rationale: `Judge failed; deterministic checks remain authoritative. ${err.message}`,
276
288
  raw: null,
277
289
  };
278
290
  }
@@ -99,16 +99,12 @@ function isEmbedderAvailable() {
99
99
  const cfg = resolveGeminiEmbeddingConfig();
100
100
  if (cfg && cfg.enabled && cfg.apiKey) return true;
101
101
  } catch { /* policy module unavailable */ }
102
- // Local transformers path
103
- try {
104
- require.resolve('@huggingface/transformers');
105
- return true;
106
- } catch { /* not installed */ }
107
- return false;
102
+ // The zero-dependency feature-hash provider is always available locally.
103
+ return true;
108
104
  }
109
105
 
110
106
  function defaultEmbedder() {
111
- // Lazy: do not pull in LanceDB/transformers at module require time.
107
+ // Lazy: do not pull in LanceDB or optional embedding providers at module require time.
112
108
  const { embed } = require('./vector-store');
113
109
  return embed;
114
110
  }