troxy-cli 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/troxy.js +16 -0
- package/package.json +1 -1
- package/src/api.js +5 -0
- package/src/hook-report.js +164 -0
- package/src/init.js +114 -2
- package/src/mcp-server.js +33 -4
- package/src/tests/claude-code-hook.test.js +125 -0
- package/src/tests/hook-report.test.js +139 -0
- package/src/tests/model-checkpoint.test.js +79 -0
package/bin/troxy.js
CHANGED
|
@@ -177,6 +177,17 @@ switch (command) {
|
|
|
177
177
|
await runMcp();
|
|
178
178
|
break;
|
|
179
179
|
|
|
180
|
+
// ── Claude Code Stop hook: real, host-captured usage per turn ─
|
|
181
|
+
// Invoked by Claude Code itself (registered via patchClaudeCodeHooks in
|
|
182
|
+
// init.js), never by a user directly. Must never throw or hang - see
|
|
183
|
+
// hook-report.js's own top-of-file note on why everything in it fails
|
|
184
|
+
// silently.
|
|
185
|
+
case 'hook-report': {
|
|
186
|
+
const { runHookReport } = await import('../src/hook-report.js');
|
|
187
|
+
await runHookReport();
|
|
188
|
+
break;
|
|
189
|
+
}
|
|
190
|
+
|
|
180
191
|
// ── Heartbeat daemon (background service) ─────────────────────
|
|
181
192
|
case 'daemon': {
|
|
182
193
|
const { runDaemon } = await import('../src/daemon.js');
|
|
@@ -532,6 +543,11 @@ switch (command) {
|
|
|
532
543
|
console.log(`
|
|
533
544
|
Troxy: AI payment control
|
|
534
545
|
|
|
546
|
+
This CLI has two parts:
|
|
547
|
+
- Commands below (policies, mcps, activity, ...) are for YOU to run and manage your account.
|
|
548
|
+
- "troxy mcp" starts the MCP server an AI agent connects to. You don't run it by hand;
|
|
549
|
+
MCP clients (Claude Desktop, etc.) launch it after "troxy init" configures them.
|
|
550
|
+
|
|
535
551
|
First time? Run these two commands in order:
|
|
536
552
|
1) npx troxy-cli init --key txy-... (get key from https://dash.troxy.io)
|
|
537
553
|
2) troxy login (start a 12h CLI session)
|
package/package.json
CHANGED
package/src/api.js
CHANGED
|
@@ -95,6 +95,11 @@ export const api = {
|
|
|
95
95
|
// guess, and Token Optimization says so on every agent that never reports.
|
|
96
96
|
evaluateModel: (body, apiKey) => request('POST', '/evaluate/model', { apiKey, body }),
|
|
97
97
|
reportModelUsage: (body, apiKey) => request('POST', '/evaluate/model/complete', { apiKey, body }),
|
|
98
|
+
// Real, host-captured usage for a turn that already happened - not an
|
|
99
|
+
// agent's self-report of what it thinks it used. Called only by the
|
|
100
|
+
// Claude Code Stop hook (hook-report.js), never by mcp-server.js's own
|
|
101
|
+
// instruction-driven self-report path.
|
|
102
|
+
reportVerifiedUsage: (body, apiKey) => request('POST', '/evaluate/model/verified', { apiKey, body }),
|
|
98
103
|
selfRevoke: (apiKey) => request('POST', '/mcp/self-revoke', { apiKey }),
|
|
99
104
|
revokeAllOthers: (apiKey) => request('POST', '/mcp/revoke-all-others', { apiKey }),
|
|
100
105
|
confirmPayment: (auditId, body, apiKey) => request('POST', `/payments/${auditId}/confirm`, { apiKey, body }),
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import { loadConfig } from './config.js';
|
|
3
|
+
import { api } from './api.js';
|
|
4
|
+
|
|
5
|
+
// Claude Code Stop-hook entry point: `troxy hook-report` (wired in bin/troxy.js).
|
|
6
|
+
// Registered by patchClaudeCodeHooks() in init.js, fires unconditionally at
|
|
7
|
+
// the end of every Claude Code turn, tool-use or pure chat, regardless of
|
|
8
|
+
// what the agent does or doesn't call. This exists because chat-only
|
|
9
|
+
// self-report was proven live to not fire reliably (two consecutive
|
|
10
|
+
// pure-chat turns with a connected agent produced zero checkpoint calls) -
|
|
11
|
+
// nothing here trusts self-report to have already happened.
|
|
12
|
+
//
|
|
13
|
+
// Everything in this file must fail silently. A broken report must never
|
|
14
|
+
// interrupt the user's Claude Code session - there is nothing more important
|
|
15
|
+
// happening here than the conversation that just finished.
|
|
16
|
+
|
|
17
|
+
const REPORT_TIMEOUT_MS = 2500;
|
|
18
|
+
|
|
19
|
+
// Fields this file is allowed to touch from a transcript entry. Deliberately
|
|
20
|
+
// narrow: extractUsage() below only ever reads these keys off a transcript
|
|
21
|
+
// line, never the line's own `content` (the actual message text). This is
|
|
22
|
+
// the enforcement mechanism for "the hook must never log, store, or forward
|
|
23
|
+
// conversation content, only usage numbers" - a function that structurally
|
|
24
|
+
// cannot see `content` cannot leak it, whether by a future edit forgetting
|
|
25
|
+
// the boundary or otherwise.
|
|
26
|
+
export function extractUsage(entry) {
|
|
27
|
+
const msg = entry?.message;
|
|
28
|
+
if (!msg || entry.type !== 'assistant') return null;
|
|
29
|
+
const usage = msg.usage;
|
|
30
|
+
if (!usage) return null;
|
|
31
|
+
const inputTokens = Number(usage.input_tokens) || 0;
|
|
32
|
+
const outputTokens = Number(usage.output_tokens) || 0;
|
|
33
|
+
const cacheCreation = Number(usage.cache_creation_input_tokens) || 0;
|
|
34
|
+
const cacheRead = Number(usage.cache_read_input_tokens) || 0;
|
|
35
|
+
return {
|
|
36
|
+
messageId: msg.id || null,
|
|
37
|
+
model: msg.model || null,
|
|
38
|
+
// Cache tokens are real spend (cache writes especially), so they count
|
|
39
|
+
// toward the total even though they're priced differently than a fresh
|
|
40
|
+
// input token. Troxy's own pricing table (chat_tools.py) does the actual
|
|
41
|
+
// dollar math server-side from the model id; this file only sums what
|
|
42
|
+
// the transcript says was used.
|
|
43
|
+
tokens: inputTokens + outputTokens + cacheCreation + cacheRead,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Claude Code writes one JSONL line per content block (thinking/tool_use/
|
|
48
|
+
// text), not one per turn or even one per model call - a single real API
|
|
49
|
+
// response is split across several consecutive lines that all share the
|
|
50
|
+
// same message.id and carry the identical usage object repeated on every
|
|
51
|
+
// line. Verified live against a real session transcript before writing
|
|
52
|
+
// this: naively summing every assistant line's usage overcounts by however
|
|
53
|
+
// many content blocks that response happened to have. A turn can also span
|
|
54
|
+
// several distinct real model calls if it used tools in between, so the fix
|
|
55
|
+
// is not "take the last line's usage" either - it's dedup by message.id and
|
|
56
|
+
// sum each unique id exactly once, walking back only as far as the last
|
|
57
|
+
// user-authored line (the start of the current turn).
|
|
58
|
+
export function usageForCurrentTurn(transcriptPath) {
|
|
59
|
+
let lines;
|
|
60
|
+
try {
|
|
61
|
+
lines = fs.readFileSync(transcriptPath, 'utf8').split('\n').filter(Boolean);
|
|
62
|
+
} catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const turnLines = [];
|
|
67
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
68
|
+
let entry;
|
|
69
|
+
try {
|
|
70
|
+
entry = JSON.parse(lines[i]);
|
|
71
|
+
} catch {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (entry.type === 'user') break;
|
|
75
|
+
turnLines.push(entry);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const seen = new Map(); // messageId -> {model, tokens}
|
|
79
|
+
let lastModel = null;
|
|
80
|
+
for (const entry of turnLines) {
|
|
81
|
+
const usage = extractUsage(entry);
|
|
82
|
+
if (!usage || !usage.messageId) continue;
|
|
83
|
+
if (!seen.has(usage.messageId)) {
|
|
84
|
+
seen.set(usage.messageId, usage);
|
|
85
|
+
lastModel = lastModel || usage.model;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (seen.size === 0) return null;
|
|
90
|
+
|
|
91
|
+
let totalTokens = 0;
|
|
92
|
+
for (const u of seen.values()) totalTokens += u.tokens;
|
|
93
|
+
|
|
94
|
+
// turn_key: dedups a hook that fires twice for the same turn (a retry,
|
|
95
|
+
// a Claude Code internal replay) against a repeat POST for the same
|
|
96
|
+
// data, and against a self-reported row for the same turn if one exists.
|
|
97
|
+
// The sorted, joined set of message ids in this turn is stable across a
|
|
98
|
+
// repeat firing of the hook for the same turn, and distinct across
|
|
99
|
+
// genuinely different turns.
|
|
100
|
+
const turnKey = Array.from(seen.keys()).sort().join(',').slice(0, 200);
|
|
101
|
+
|
|
102
|
+
return { tokens: totalTokens, model: lastModel, turnKey };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function withTimeout(promise, ms) {
|
|
106
|
+
return Promise.race([
|
|
107
|
+
promise,
|
|
108
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), ms)),
|
|
109
|
+
]);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function readStdin() {
|
|
113
|
+
const chunks = [];
|
|
114
|
+
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
115
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export async function runHookReport() {
|
|
119
|
+
try {
|
|
120
|
+
const raw = await withTimeout(readStdin(), 1000);
|
|
121
|
+
let payload;
|
|
122
|
+
try {
|
|
123
|
+
payload = JSON.parse(raw);
|
|
124
|
+
} catch {
|
|
125
|
+
process.exit(0);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const transcriptPath = payload?.transcript_path;
|
|
129
|
+
if (!transcriptPath) {
|
|
130
|
+
process.exit(0);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const usage = usageForCurrentTurn(transcriptPath);
|
|
134
|
+
if (!usage || !usage.model || usage.tokens <= 0) {
|
|
135
|
+
// Nothing measurable this turn (e.g. the very first line of a fresh
|
|
136
|
+
// session, or a turn that produced no assistant usage at all) - not
|
|
137
|
+
// an error, just nothing to report.
|
|
138
|
+
process.exit(0);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const config = loadConfig();
|
|
142
|
+
const apiKey = process.env.TROXY_API_KEY || config?.apiKey;
|
|
143
|
+
if (!apiKey) {
|
|
144
|
+
process.exit(0);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const sessionId = payload?.session_id || '';
|
|
148
|
+
const turnKey = sessionId ? `${sessionId}:${usage.turnKey}` : null;
|
|
149
|
+
|
|
150
|
+
await withTimeout(
|
|
151
|
+
api.reportVerifiedUsage(
|
|
152
|
+
{ model: usage.model, actual_tokens: usage.tokens, turn_key: turnKey },
|
|
153
|
+
apiKey,
|
|
154
|
+
),
|
|
155
|
+
REPORT_TIMEOUT_MS,
|
|
156
|
+
);
|
|
157
|
+
} catch {
|
|
158
|
+
// Never surface a failure to Claude Code. A dropped report is a gap in
|
|
159
|
+
// Token Optimization, not something worth interrupting the user's
|
|
160
|
+
// session over.
|
|
161
|
+
} finally {
|
|
162
|
+
process.exit(0);
|
|
163
|
+
}
|
|
164
|
+
}
|
package/src/init.js
CHANGED
|
@@ -28,7 +28,12 @@ export function detectMcpClients() {
|
|
|
28
28
|
hasOpenClaw = true;
|
|
29
29
|
} catch {}
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
const hasClaude = hasClaudeCode();
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
detected, hasOpenClaw, hasClaude,
|
|
35
|
+
none: detected.length === 0 && !hasOpenClaw && !hasClaude,
|
|
36
|
+
};
|
|
32
37
|
}
|
|
33
38
|
|
|
34
39
|
// `troxy update` on a hosted agent. The npm package is beside the point there:
|
|
@@ -44,7 +49,7 @@ export async function refreshHostedInstructions(key) {
|
|
|
44
49
|
|
|
45
50
|
export async function reprovisionKeyConsumers(key, agentName) {
|
|
46
51
|
const platform = process.platform;
|
|
47
|
-
const { detected, hasOpenClaw, none } = detectMcpClients();
|
|
52
|
+
const { detected, hasOpenClaw, hasClaude, none } = detectMcpClients();
|
|
48
53
|
|
|
49
54
|
if (none) {
|
|
50
55
|
// No MCP client on this machine almost always means a hosted agent running
|
|
@@ -82,6 +87,20 @@ export async function reprovisionKeyConsumers(key, agentName) {
|
|
|
82
87
|
console.log(` • OpenClaw ✗ (${err.message})`);
|
|
83
88
|
}
|
|
84
89
|
}
|
|
90
|
+
if (hasClaude) {
|
|
91
|
+
try {
|
|
92
|
+
patchClaudeCodeConfig(claudeCodeConfigPath(), key);
|
|
93
|
+
console.log(` • Claude Code (this project) ✓`);
|
|
94
|
+
} catch (err) {
|
|
95
|
+
console.log(` • Claude Code ✗ (${err.message})`);
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
patchClaudeCodeHooks(claudeCodeSettingsPath());
|
|
99
|
+
console.log(` • Claude Code usage capture (all projects) ✓`);
|
|
100
|
+
} catch (err) {
|
|
101
|
+
console.log(` • Claude Code usage capture ✗ (${err.message})`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
85
104
|
console.log('\n Restart your MCP client to activate Troxy.');
|
|
86
105
|
}
|
|
87
106
|
|
|
@@ -419,3 +438,96 @@ function patchContinueConfig(configPath, apiKey) {
|
|
|
419
438
|
});
|
|
420
439
|
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
|
|
421
440
|
}
|
|
441
|
+
|
|
442
|
+
// Claude Code does not use the flat `mcpServers` file the other five clients
|
|
443
|
+
// share - its MCP registration is project-scoped, living in `~/.claude.json`
|
|
444
|
+
// under `projects["<cwd>"].mcpServers`, keyed by the directory `troxy init`
|
|
445
|
+
// was run from. Confirmed by reading a real, populated `~/.claude.json` on a
|
|
446
|
+
// machine with Claude Code installed - there is no separate, simpler config
|
|
447
|
+
// file to target. Before this, Claude Code had no entry in MCP_CLIENTS at
|
|
448
|
+
// all, so `troxy init` misdetected a Claude-Code-only machine as having no
|
|
449
|
+
// MCP client and printed hosted-agent instructions instead of registering
|
|
450
|
+
// anything.
|
|
451
|
+
//
|
|
452
|
+
// TROXY_HOOK_CAPTURE=1 is the signal mcp-server.js's own instructions check
|
|
453
|
+
// to suppress its chat-only self-report clause for this session - once the
|
|
454
|
+
// Stop hook (patchClaudeCodeHooks, below) is also registered, self-report
|
|
455
|
+
// asking the agent to also report chat-only turns would double-count against
|
|
456
|
+
// the hook's real capture of the same turns. Tool-use self-report is left
|
|
457
|
+
// alone; the hook does not replace that, only chat-only reporting.
|
|
458
|
+
function claudeCodeConfigPath() {
|
|
459
|
+
return path.join(os.homedir(), '.claude.json');
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// configPath and cwd are explicit params, same shape as patchMcpConfig /
|
|
463
|
+
// patchZedConfig above, so this can be pointed at a temp file and a fake
|
|
464
|
+
// project directory in tests rather than always touching the real
|
|
465
|
+
// ~/.claude.json on whatever machine the tests run on.
|
|
466
|
+
export function patchClaudeCodeConfig(configPath, apiKey, cwd = process.cwd()) {
|
|
467
|
+
let config = {};
|
|
468
|
+
try { config = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch {}
|
|
469
|
+
if (!config.projects) config.projects = {};
|
|
470
|
+
if (!config.projects[cwd]) config.projects[cwd] = {};
|
|
471
|
+
if (!config.projects[cwd].mcpServers) config.projects[cwd].mcpServers = {};
|
|
472
|
+
config.projects[cwd].mcpServers.troxy = {
|
|
473
|
+
command: 'npx',
|
|
474
|
+
args: ['troxy-cli', 'mcp'],
|
|
475
|
+
env: { TROXY_API_KEY: apiKey, TROXY_HOOK_CAPTURE: '1' },
|
|
476
|
+
};
|
|
477
|
+
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// The hook, unlike MCP registration, is written to the GLOBAL user settings
|
|
481
|
+
// file rather than a per-project one: registered once, it captures real
|
|
482
|
+
// usage across every project this user works in with Claude Code, not just
|
|
483
|
+
// the one `troxy init` happened to run inside. Idempotent by finding any
|
|
484
|
+
// existing Stop hook entry whose command already invokes `troxy hook-report`
|
|
485
|
+
// and replacing it in place, rather than blind-pushing - a second `troxy
|
|
486
|
+
// init` must not duplicate the entry, and any of the user's OWN unrelated
|
|
487
|
+
// Stop hooks in the same file must survive untouched.
|
|
488
|
+
function claudeCodeSettingsPath() {
|
|
489
|
+
return path.join(os.homedir(), '.claude', 'settings.json');
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
export function troxyHookCommand() {
|
|
493
|
+
let troxy;
|
|
494
|
+
try {
|
|
495
|
+
troxy = execSync('which troxy').toString().trim();
|
|
496
|
+
} catch {
|
|
497
|
+
troxy = 'npx troxy-cli';
|
|
498
|
+
}
|
|
499
|
+
return `${troxy} hook-report`;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
export function patchClaudeCodeHooks(configPath, command = troxyHookCommand()) {
|
|
503
|
+
let config = {};
|
|
504
|
+
try { config = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch {}
|
|
505
|
+
if (!config.hooks) config.hooks = {};
|
|
506
|
+
if (!Array.isArray(config.hooks.Stop)) config.hooks.Stop = [];
|
|
507
|
+
|
|
508
|
+
const isTroxyEntry = (matcherEntry) =>
|
|
509
|
+
Array.isArray(matcherEntry?.hooks) &&
|
|
510
|
+
matcherEntry.hooks.some(h => typeof h?.command === 'string' && h.command.includes('hook-report'));
|
|
511
|
+
|
|
512
|
+
const troxyEntry = { hooks: [{ type: 'command', command }] };
|
|
513
|
+
const idx = config.hooks.Stop.findIndex(isTroxyEntry);
|
|
514
|
+
if (idx >= 0) config.hooks.Stop[idx] = troxyEntry;
|
|
515
|
+
else config.hooks.Stop.push(troxyEntry);
|
|
516
|
+
|
|
517
|
+
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
|
518
|
+
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// Detects Claude Code the same way the daemon detects OpenClaw: by whether
|
|
522
|
+
// its own CLI is on PATH, since (unlike the other five clients) there is no
|
|
523
|
+
// single marker *file* whose mere existence means "Claude Code is
|
|
524
|
+
// installed" - `~/.claude.json` and `~/.claude/settings.json` are both
|
|
525
|
+
// created lazily and may not exist yet on a fresh install.
|
|
526
|
+
function hasClaudeCode() {
|
|
527
|
+
try {
|
|
528
|
+
execSync('claude --version', { stdio: 'ignore' });
|
|
529
|
+
return true;
|
|
530
|
+
} catch {
|
|
531
|
+
return false;
|
|
532
|
+
}
|
|
533
|
+
}
|
package/src/mcp-server.js
CHANGED
|
@@ -31,6 +31,18 @@ export async function runMcp() {
|
|
|
31
31
|
process.exit(1);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
// TROXY_HOOK_CAPTURE is set only by patchClaudeCodeConfig (init.js), only
|
|
35
|
+
// for a Claude Code session that also has the Stop hook registered
|
|
36
|
+
// (patchClaudeCodeHooks). That hook reports real, host-captured usage for
|
|
37
|
+
// every turn regardless of what the agent does - so instructing the agent
|
|
38
|
+
// to ALSO self-report chat-only turns here would double-count the same
|
|
39
|
+
// turn once as a self-report and once as a verified capture. Tool-use
|
|
40
|
+
// self-report is untouched: the hook does not gate on a policy decision
|
|
41
|
+
// (it fires after the turn already happened, there's nothing left to
|
|
42
|
+
// block), so it is not a substitute for the pre-call gating self-report
|
|
43
|
+
// still provides.
|
|
44
|
+
const hookCaptureActive = !!process.env.TROXY_HOOK_CAPTURE;
|
|
45
|
+
|
|
34
46
|
const server = new Server(
|
|
35
47
|
{
|
|
36
48
|
name: 'troxy',
|
|
@@ -62,7 +74,22 @@ export async function runMcp() {
|
|
|
62
74
|
// Token Optimization is supposed to be a precise record, not a mood.
|
|
63
75
|
// The trigger below is mechanical instead: did this turn use a tool,
|
|
64
76
|
// yes or no. That is a fact you can check, not a judgment you make.
|
|
65
|
-
'You MUST also call the evaluate_model tool at the start of ANY turn where you will use one or more tools of any kind — reading or writing a file, running a command, browsing, calling an external API, anything beyond a plain text reply — and whenever you switch model or effort level partway through the turn. This is not a judgment call about how the turn feels: if you are about to use a tool, call evaluate_model first, every time, regardless of how small the turn seems
|
|
77
|
+
'You MUST also call the evaluate_model tool at the start of ANY turn where you will use one or more tools of any kind — reading or writing a file, running a command, browsing, calling an external API, anything beyond a plain text reply — and whenever you switch model or effort level partway through the turn. This is not a judgment call about how the turn feels: if you are about to use a tool, call evaluate_model first, every time, regardless of how small the turn seems, and WAIT for the decision before proceeding. Give the exact model id and how many tokens you expect to use. Do not use any tool without first receiving an ALLOW or NOTIFY decision; if BLOCK, pick a cheaper or smaller model and call evaluate_model again with the new id; if ESCALATE, call wait_for_approval and do not proceed until it resolves. Then, at the end of that same turn, you MUST call report_model_usage with the audit_id from that decision and the real token total from your model API. Both halves are required: without the second call Troxy has only your prediction, and the account owner is shown that this agent\'s costs are unverified. Report the true number even when it is far above your estimate. ' +
|
|
78
|
+
(hookCaptureActive
|
|
79
|
+
// Claude Code's own Stop hook already reports real, host-captured
|
|
80
|
+
// usage for every turn in this session, chat-only included - see
|
|
81
|
+
// the note above `hookCaptureActive`. Self-reporting chat here too
|
|
82
|
+
// would double-count the same turn.
|
|
83
|
+
? 'This session has real usage capture active for every turn (Claude Code\'s own Stop hook, not a self-report), so a turn that is pure conversation with no tool use is already covered - do NOT call evaluate_model or report_model_usage for it yourself. Tool-use turns are unaffected: keep gating on evaluate_model before using any tool exactly as described above.'
|
|
84
|
+
// A turn that is PURE conversation, no tool use at all, still gets
|
|
85
|
+
// logged, just not gated. Blocking a chat reply on a network round
|
|
86
|
+
// trip protects against a risk that is vanishingly rare in practice
|
|
87
|
+
// (almost nobody writes a policy to block message content), and the
|
|
88
|
+
// cost of blocking anyway is a slower product on every single message.
|
|
89
|
+
// So chat calls the same two tools, in the same order, but treats the
|
|
90
|
+
// decision as a record rather than a gate: reply to the user first,
|
|
91
|
+
// report afterward, never hold the reply for Troxy's answer.
|
|
92
|
+
: 'A turn that is pure conversation, with no tool use at all, is different: you still call evaluate_model and report_model_usage for it, but you do NOT wait for or act on the decision before replying. Reply to the user first. Then, as the last step of that same turn (or the first step of your next turn if your environment does not let you act again after producing a reply), call evaluate_model with the model and your best token estimate, and call report_model_usage with the real total right after — same two tools as above, just not gating. If the decision comes back BLOCK or ESCALATE for a chat turn, do not retry with a different model and do not poll wait_for_approval: the reply has already been sent, there is nothing left to gate, and the decision is recorded for the account owner\'s visibility rather than enforced in the moment. Chat usage still counts toward every rolling spend and token limit, so it still shapes what the next tool-use call is allowed to do.'),
|
|
66
93
|
},
|
|
67
94
|
{ capabilities: { tools: {} } },
|
|
68
95
|
);
|
|
@@ -279,9 +306,11 @@ export async function runMcp() {
|
|
|
279
306
|
description:
|
|
280
307
|
'Evaluate whether running a task on a given AI model should be allowed, blocked, or escalated ' +
|
|
281
308
|
'based on your Troxy policies, and record what it is expected to cost. ' +
|
|
282
|
-
'Call this at the start
|
|
283
|
-
|
|
284
|
-
|
|
309
|
+
'Call this on every turn: at the start, before proceeding, if you will use a tool of any kind, or whenever you switch model or effort level mid-turn' +
|
|
310
|
+
(hookCaptureActive
|
|
311
|
+
? '. This session has real usage capture active for chat-only turns already (do not call this or report_model_usage for a turn with no tool use - only for tool-use turns). '
|
|
312
|
+
: '; or after replying, without waiting for the decision, if the turn was pure conversation with no tool use. Tool use gates on the decision; chat only records it. ') +
|
|
313
|
+
'Returns an audit_id. Keep it: you must pass it to report_model_usage when the turn finishes.',
|
|
285
314
|
inputSchema: {
|
|
286
315
|
type: 'object',
|
|
287
316
|
required: ['model'],
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// Claude Code has no entry in MCP_CLIENTS at all before this change - on a
|
|
2
|
+
// machine with only Claude Code installed, `troxy init` misdetected "no MCP
|
|
3
|
+
// client" and printed hosted-agent instructions instead of registering
|
|
4
|
+
// anything. Its MCP config also has a different shape than the other five
|
|
5
|
+
// clients: project-scoped, under `~/.claude.json`'s `projects["<cwd>"]`,
|
|
6
|
+
// confirmed by reading a real populated file rather than assumed from docs.
|
|
7
|
+
//
|
|
8
|
+
// The Stop hook exists to close a gap self-report cannot: it fires
|
|
9
|
+
// unconditionally at the end of every turn (tool-use or pure chat) and
|
|
10
|
+
// reads real usage off Claude Code's own transcript, regardless of what the
|
|
11
|
+
// agent does. Chat-only self-report was proven live to not fire reliably;
|
|
12
|
+
// this does not depend on the agent choosing to comply at all.
|
|
13
|
+
|
|
14
|
+
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
15
|
+
import assert from 'node:assert/strict';
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import os from 'node:os';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
|
|
20
|
+
import { patchClaudeCodeConfig, patchClaudeCodeHooks } from '../init.js';
|
|
21
|
+
|
|
22
|
+
let dir;
|
|
23
|
+
beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'troxy-cc-')); });
|
|
24
|
+
afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
|
|
25
|
+
|
|
26
|
+
describe('patchClaudeCodeConfig', () => {
|
|
27
|
+
const configPath = () => path.join(dir, '.claude.json');
|
|
28
|
+
const read = () => JSON.parse(fs.readFileSync(configPath(), 'utf8'));
|
|
29
|
+
|
|
30
|
+
it('creates the project-scoped mcpServers entry when the file does not exist', () => {
|
|
31
|
+
patchClaudeCodeConfig(configPath(), 'txy-abc', '/Users/x/proj');
|
|
32
|
+
const cfg = read();
|
|
33
|
+
assert.equal(cfg.projects['/Users/x/proj'].mcpServers.troxy.env.TROXY_API_KEY, 'txy-abc');
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('sets TROXY_HOOK_CAPTURE so mcp-server.js knows the hook already covers chat', () => {
|
|
37
|
+
patchClaudeCodeConfig(configPath(), 'txy-abc', '/Users/x/proj');
|
|
38
|
+
const cfg = read();
|
|
39
|
+
assert.equal(cfg.projects['/Users/x/proj'].mcpServers.troxy.env.TROXY_HOOK_CAPTURE, '1');
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('only touches this project entry, leaving other projects in the same file alone', () => {
|
|
43
|
+
fs.writeFileSync(configPath(), JSON.stringify({
|
|
44
|
+
projects: { '/other/proj': { mcpServers: { someOtherTool: { command: 'x' } } } },
|
|
45
|
+
}));
|
|
46
|
+
patchClaudeCodeConfig(configPath(), 'txy-abc', '/Users/x/proj');
|
|
47
|
+
const cfg = read();
|
|
48
|
+
assert.ok(cfg.projects['/other/proj'].mcpServers.someOtherTool, 'unrelated project entry was dropped');
|
|
49
|
+
assert.ok(cfg.projects['/Users/x/proj'].mcpServers.troxy);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('preserves unrelated keys already on the project entry (e.g. allowedTools)', () => {
|
|
53
|
+
fs.writeFileSync(configPath(), JSON.stringify({
|
|
54
|
+
projects: { '/Users/x/proj': { allowedTools: ['Bash'], mcpServers: {} } },
|
|
55
|
+
}));
|
|
56
|
+
patchClaudeCodeConfig(configPath(), 'txy-abc', '/Users/x/proj');
|
|
57
|
+
const cfg = read();
|
|
58
|
+
assert.deepEqual(cfg.projects['/Users/x/proj'].allowedTools, ['Bash']);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('re-running with a new key overwrites the old one rather than stacking', () => {
|
|
62
|
+
patchClaudeCodeConfig(configPath(), 'txy-old', '/Users/x/proj');
|
|
63
|
+
patchClaudeCodeConfig(configPath(), 'txy-new', '/Users/x/proj');
|
|
64
|
+
const cfg = read();
|
|
65
|
+
assert.equal(cfg.projects['/Users/x/proj'].mcpServers.troxy.env.TROXY_API_KEY, 'txy-new');
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it('does not throw when the existing file is corrupted JSON', () => {
|
|
69
|
+
fs.writeFileSync(configPath(), '{ not json');
|
|
70
|
+
assert.doesNotThrow(() => patchClaudeCodeConfig(configPath(), 'txy-abc', '/Users/x/proj'));
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe('patchClaudeCodeHooks', () => {
|
|
75
|
+
const configPath = () => path.join(dir, 'settings.json');
|
|
76
|
+
const read = () => JSON.parse(fs.readFileSync(configPath(), 'utf8'));
|
|
77
|
+
|
|
78
|
+
it('creates the Stop hook entry when the file does not exist', () => {
|
|
79
|
+
patchClaudeCodeHooks(configPath(), '/usr/local/bin/troxy hook-report');
|
|
80
|
+
const cfg = read();
|
|
81
|
+
assert.equal(cfg.hooks.Stop.length, 1);
|
|
82
|
+
assert.equal(cfg.hooks.Stop[0].hooks[0].command, '/usr/local/bin/troxy hook-report');
|
|
83
|
+
assert.equal(cfg.hooks.Stop[0].hooks[0].type, 'command');
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('is idempotent: running twice produces one Troxy entry, not two', () => {
|
|
87
|
+
patchClaudeCodeHooks(configPath(), 'troxy hook-report');
|
|
88
|
+
patchClaudeCodeHooks(configPath(), 'troxy hook-report');
|
|
89
|
+
const cfg = read();
|
|
90
|
+
assert.equal(cfg.hooks.Stop.length, 1);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it('updates the command in place on a re-run rather than appending', () => {
|
|
94
|
+
patchClaudeCodeHooks(configPath(), 'npx troxy-cli hook-report');
|
|
95
|
+
patchClaudeCodeHooks(configPath(), '/usr/local/bin/troxy hook-report');
|
|
96
|
+
const cfg = read();
|
|
97
|
+
assert.equal(cfg.hooks.Stop.length, 1);
|
|
98
|
+
assert.equal(cfg.hooks.Stop[0].hooks[0].command, '/usr/local/bin/troxy hook-report');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("preserves the user's own unrelated Stop hooks in the same file", () => {
|
|
102
|
+
fs.writeFileSync(configPath(), JSON.stringify({
|
|
103
|
+
hooks: { Stop: [{ hooks: [{ type: 'command', command: 'my-own-script.sh' }] }] },
|
|
104
|
+
}));
|
|
105
|
+
patchClaudeCodeHooks(configPath(), 'troxy hook-report');
|
|
106
|
+
const cfg = read();
|
|
107
|
+
assert.equal(cfg.hooks.Stop.length, 2);
|
|
108
|
+
assert.ok(cfg.hooks.Stop.some(e => e.hooks[0].command === 'my-own-script.sh'));
|
|
109
|
+
assert.ok(cfg.hooks.Stop.some(e => e.hooks[0].command === 'troxy hook-report'));
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("preserves the user's own other hook events (PreToolUse etc.) untouched", () => {
|
|
113
|
+
fs.writeFileSync(configPath(), JSON.stringify({
|
|
114
|
+
hooks: { PreToolUse: [{ matcher: 'Write', hooks: [{ type: 'command', command: 'validate.sh' }] }] },
|
|
115
|
+
}));
|
|
116
|
+
patchClaudeCodeHooks(configPath(), 'troxy hook-report');
|
|
117
|
+
const cfg = read();
|
|
118
|
+
assert.ok(cfg.hooks.PreToolUse, 'an unrelated hook event was dropped by the patch');
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('does not throw when the existing file is corrupted JSON', () => {
|
|
122
|
+
fs.writeFileSync(configPath(), 'not json at all');
|
|
123
|
+
assert.doesNotThrow(() => patchClaudeCodeHooks(configPath(), 'troxy hook-report'));
|
|
124
|
+
});
|
|
125
|
+
});
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// The transcript-parsing half of the Claude Code Stop hook.
|
|
2
|
+
//
|
|
3
|
+
// Verified live against a real session transcript before writing this (see
|
|
4
|
+
// the plan): Claude Code writes one JSONL line per content block
|
|
5
|
+
// (thinking/tool_use/text), not one per turn or even one per real model
|
|
6
|
+
// call. Multiple consecutive lines share the same message.id and top-level
|
|
7
|
+
// requestId, and carry the IDENTICAL usage object repeated on every line.
|
|
8
|
+
// Naively summing every assistant line's usage overcounts by however many
|
|
9
|
+
// content blocks that one response happened to have - these tests exist
|
|
10
|
+
// because that bug is easy to reintroduce and would silently inflate every
|
|
11
|
+
// number Token Optimization shows for a Claude Code agent.
|
|
12
|
+
|
|
13
|
+
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
14
|
+
import assert from 'node:assert/strict';
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import os from 'node:os';
|
|
17
|
+
import path from 'node:path';
|
|
18
|
+
|
|
19
|
+
import { usageForCurrentTurn, extractUsage } from '../hook-report.js';
|
|
20
|
+
|
|
21
|
+
let dir;
|
|
22
|
+
beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'troxy-transcript-')); });
|
|
23
|
+
afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
|
|
24
|
+
|
|
25
|
+
function transcriptPath() {
|
|
26
|
+
return path.join(dir, 'transcript.jsonl');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function writeLines(lines) {
|
|
30
|
+
fs.writeFileSync(transcriptPath(), lines.map(l => JSON.stringify(l)).join('\n') + '\n');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function assistantLine(messageId, usage, model = 'claude-sonnet-5') {
|
|
34
|
+
return { type: 'assistant', message: { id: messageId, model, usage } };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const USAGE_A = { input_tokens: 2, output_tokens: 1176, cache_creation_input_tokens: 4134, cache_read_input_tokens: 244103 };
|
|
38
|
+
|
|
39
|
+
describe('extractUsage', () => {
|
|
40
|
+
it('only reads usage/model/id fields, never message.content', () => {
|
|
41
|
+
const entry = {
|
|
42
|
+
type: 'assistant',
|
|
43
|
+
message: {
|
|
44
|
+
id: 'msg_1',
|
|
45
|
+
model: 'claude-sonnet-5',
|
|
46
|
+
usage: { input_tokens: 10, output_tokens: 5 },
|
|
47
|
+
content: [{ type: 'text', text: 'this is the actual conversation, must never be touched' }],
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
const result = extractUsage(entry);
|
|
51
|
+
assert.deepEqual(Object.keys(result).sort(), ['messageId', 'model', 'tokens']);
|
|
52
|
+
assert.ok(!JSON.stringify(result).includes('actual conversation'));
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('returns null for a non-assistant line', () => {
|
|
56
|
+
assert.equal(extractUsage({ type: 'user', message: { usage: { input_tokens: 1 } } }), null);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('returns null when there is no usage object at all', () => {
|
|
60
|
+
assert.equal(extractUsage({ type: 'assistant', message: { id: 'x' } }), null);
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe('usageForCurrentTurn', () => {
|
|
65
|
+
it('sums usage once per unique message.id, not once per JSONL line', () => {
|
|
66
|
+
// The exact shape found live: three content-block lines (thinking,
|
|
67
|
+
// tool_use, text) all sharing one message.id and one usage object.
|
|
68
|
+
writeLines([
|
|
69
|
+
{ type: 'user', message: { content: [{ type: 'text', text: 'hi' }] } },
|
|
70
|
+
assistantLine('msg_1', USAGE_A),
|
|
71
|
+
assistantLine('msg_1', USAGE_A),
|
|
72
|
+
assistantLine('msg_1', USAGE_A),
|
|
73
|
+
]);
|
|
74
|
+
const result = usageForCurrentTurn(transcriptPath());
|
|
75
|
+
const expectedOnce = USAGE_A.input_tokens + USAGE_A.output_tokens
|
|
76
|
+
+ USAGE_A.cache_creation_input_tokens + USAGE_A.cache_read_input_tokens;
|
|
77
|
+
assert.equal(result.tokens, expectedOnce, 'summed the same message.id multiple times');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('sums each distinct message.id once when a turn used a tool mid-turn (two real model calls)', () => {
|
|
81
|
+
writeLines([
|
|
82
|
+
{ type: 'user', message: { content: [{ type: 'text', text: 'hi' }] } },
|
|
83
|
+
assistantLine('msg_1', { input_tokens: 10, output_tokens: 20 }),
|
|
84
|
+
assistantLine('msg_1', { input_tokens: 10, output_tokens: 20 }), // same call, second content block
|
|
85
|
+
assistantLine('msg_2', { input_tokens: 5, output_tokens: 8 }), // a second real model call after a tool result
|
|
86
|
+
]);
|
|
87
|
+
const result = usageForCurrentTurn(transcriptPath());
|
|
88
|
+
assert.equal(result.tokens, 10 + 20 + 5 + 8);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('only looks back to the last user-authored line, not the whole session', () => {
|
|
92
|
+
writeLines([
|
|
93
|
+
{ type: 'user', message: { content: [{ type: 'text', text: 'first turn' }] } },
|
|
94
|
+
assistantLine('msg_old', { input_tokens: 999, output_tokens: 999 }),
|
|
95
|
+
{ type: 'user', message: { content: [{ type: 'text', text: 'second turn' }] } },
|
|
96
|
+
assistantLine('msg_new', { input_tokens: 10, output_tokens: 10 }),
|
|
97
|
+
]);
|
|
98
|
+
const result = usageForCurrentTurn(transcriptPath());
|
|
99
|
+
assert.equal(result.tokens, 20, 'usage from a prior turn leaked into the current one');
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('returns the model from the assistant lines in the current turn', () => {
|
|
103
|
+
writeLines([
|
|
104
|
+
{ type: 'user', message: { content: [{ type: 'text', text: 'hi' }] } },
|
|
105
|
+
assistantLine('msg_1', { input_tokens: 1, output_tokens: 1 }, 'claude-opus-4.5'),
|
|
106
|
+
]);
|
|
107
|
+
const result = usageForCurrentTurn(transcriptPath());
|
|
108
|
+
assert.equal(result.model, 'claude-opus-4.5');
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('returns a stable turn_key for the same set of message ids, order-independent', () => {
|
|
112
|
+
writeLines([
|
|
113
|
+
{ type: 'user', message: { content: [] } },
|
|
114
|
+
assistantLine('msg_b', { input_tokens: 1, output_tokens: 1 }),
|
|
115
|
+
assistantLine('msg_a', { input_tokens: 1, output_tokens: 1 }),
|
|
116
|
+
]);
|
|
117
|
+
const result = usageForCurrentTurn(transcriptPath());
|
|
118
|
+
assert.equal(result.turnKey, 'msg_a,msg_b');
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('returns null when the transcript has no assistant usage at all (fresh session)', () => {
|
|
122
|
+
writeLines([{ type: 'user', message: { content: [{ type: 'text', text: 'hi' }] } }]);
|
|
123
|
+
assert.equal(usageForCurrentTurn(transcriptPath()), null);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('does not throw when the transcript file does not exist', () => {
|
|
127
|
+
assert.doesNotThrow(() => usageForCurrentTurn(path.join(dir, 'missing.jsonl')));
|
|
128
|
+
assert.equal(usageForCurrentTurn(path.join(dir, 'missing.jsonl')), null);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('skips unparseable lines rather than throwing', () => {
|
|
132
|
+
fs.writeFileSync(transcriptPath(), [
|
|
133
|
+
JSON.stringify({ type: 'user', message: { content: [] } }),
|
|
134
|
+
'not valid json',
|
|
135
|
+
JSON.stringify(assistantLine('msg_1', { input_tokens: 1, output_tokens: 1 })),
|
|
136
|
+
].join('\n') + '\n');
|
|
137
|
+
assert.doesNotThrow(() => usageForCurrentTurn(transcriptPath()));
|
|
138
|
+
});
|
|
139
|
+
});
|
|
@@ -166,3 +166,82 @@ test('evaluate_model\'s own tool description matches the mechanical trigger', ()
|
|
|
166
166
|
assert.ok(!/substantial/i.test(desc), 'the tool description still says "substantial"');
|
|
167
167
|
assert.ok(/tool/i.test(desc), 'the tool description does not mention the tool-use trigger');
|
|
168
168
|
});
|
|
169
|
+
|
|
170
|
+
test('pure chat still reports usage, it just does not gate on the decision', () => {
|
|
171
|
+
// The first version of this fix (tool-use = mechanical trigger) left chat
|
|
172
|
+
// unreported entirely, on the reasoning that chat isn't "work." That
|
|
173
|
+
// reasoning doesn't survive contact with the goal: chat still costs real
|
|
174
|
+
// tokens, and a page that silently misses all chat-only usage is not a
|
|
175
|
+
// precise record, it just moved the same gap somewhere less visible.
|
|
176
|
+
const instructions = src.slice(src.indexOf("'You are connected to Troxy"), src.indexOf('capabilities'));
|
|
177
|
+
const chatSeg = instructions.slice(instructions.indexOf('A turn that is pure conversation'));
|
|
178
|
+
assert.ok(chatSeg, 'no instruction exists for the pure-conversation case at all');
|
|
179
|
+
assert.ok(
|
|
180
|
+
/still call evaluate_model and report_model_usage/.test(chatSeg),
|
|
181
|
+
'chat turns are not instructed to call the checkpoint tools at all, so '
|
|
182
|
+
+ 'chat-only usage is invisible on Token Optimization',
|
|
183
|
+
);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test('chat is told explicitly not to block the reply on the decision', () => {
|
|
187
|
+
// The whole point of treating chat differently from tool use is that a
|
|
188
|
+
// human is waiting on the reply. If the instructions still say "wait for
|
|
189
|
+
// the decision" for chat, this fix adds latency to every message for no
|
|
190
|
+
// behavioural gain over the tool-use path it was meant to avoid copying.
|
|
191
|
+
const instructions = src.slice(src.indexOf("'You are connected to Troxy"), src.indexOf('capabilities'));
|
|
192
|
+
const chatSeg = instructions.slice(instructions.indexOf('A turn that is pure conversation'));
|
|
193
|
+
assert.ok(
|
|
194
|
+
/do NOT wait for or act on the decision before replying/.test(chatSeg),
|
|
195
|
+
'chat instructions do not say to reply before waiting on the checkpoint',
|
|
196
|
+
);
|
|
197
|
+
assert.ok(
|
|
198
|
+
/Reply to the user first/.test(chatSeg),
|
|
199
|
+
'chat instructions do not establish reply-then-report ordering',
|
|
200
|
+
);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
test('a BLOCK or ESCALATE on a chat turn does not trigger retry or polling', () => {
|
|
204
|
+
// Retrying with a cheaper model, or polling wait_for_approval, are the
|
|
205
|
+
// tool-use responses to BLOCK/ESCALATE. For chat the reply is already sent
|
|
206
|
+
// by the time the decision comes back, so both of those actions are
|
|
207
|
+
// meaningless and would just make the agent stall on a message that
|
|
208
|
+
// already went out.
|
|
209
|
+
const instructions = src.slice(src.indexOf("'You are connected to Troxy"), src.indexOf('capabilities'));
|
|
210
|
+
const chatSeg = instructions.slice(instructions.indexOf('A turn that is pure conversation'));
|
|
211
|
+
assert.ok(
|
|
212
|
+
/do not retry with a different model and do not poll wait_for_approval/.test(chatSeg),
|
|
213
|
+
'chat instructions do not rule out the tool-use recovery actions, which '
|
|
214
|
+
+ 'would stall a reply that has already been sent',
|
|
215
|
+
);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test('chat usage still feeds the rolling limits that gate tool use', () => {
|
|
219
|
+
// Not gating chat in real time is only an acceptable trade if chat spend
|
|
220
|
+
// still counts toward spend_per_hour / tokens_per_day, so a chat-heavy
|
|
221
|
+
// agent still gets caught the moment it tries to use a tool. Otherwise
|
|
222
|
+
// "we don't block chat" quietly becomes "chat spend is uncapped."
|
|
223
|
+
const instructions = src.slice(src.indexOf("'You are connected to Troxy"), src.indexOf('capabilities'));
|
|
224
|
+
const chatSeg = instructions.slice(instructions.indexOf('A turn that is pure conversation'));
|
|
225
|
+
assert.ok(
|
|
226
|
+
/still counts toward every rolling spend and token limit/.test(chatSeg),
|
|
227
|
+
'nothing tells the agent that unblocked chat spend still accumulates '
|
|
228
|
+
+ 'toward the limits a later tool-use call is checked against',
|
|
229
|
+
);
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
test('no new tool was introduced for the chat path', () => {
|
|
233
|
+
// The backend already supports this: report_model_usage looks up its row
|
|
234
|
+
// by id/user/agent, never by decision, so it works identically whether the
|
|
235
|
+
// row says ALLOW, BLOCK, or ESCALATE. Reusing evaluate_model +
|
|
236
|
+
// report_model_usage for chat means zero backend changes and zero new
|
|
237
|
+
// surface area to get wrong; a bespoke chat-logging tool would duplicate a
|
|
238
|
+
// path that is already tested end to end.
|
|
239
|
+
// Scoped to the tools array, not the whole file: the Server constructor
|
|
240
|
+
// also has a top-level `name: 'troxy'` for the server itself, which the
|
|
241
|
+
// unscoped regex was catching as a ninth "tool".
|
|
242
|
+
const toolsSeg = src.slice(src.indexOf('tools: ['), src.indexOf('server.setRequestHandler(CallToolRequestSchema'));
|
|
243
|
+
const toolNames = [...toolsSeg.matchAll(/name: '([a-z_]+)'/g)].map(m => m[1]);
|
|
244
|
+
const unique = new Set(toolNames);
|
|
245
|
+
assert.strictEqual(unique.size, 8, 'a tool was added or removed for this change');
|
|
246
|
+
assert.ok(unique.has('evaluate_model') && unique.has('report_model_usage'));
|
|
247
|
+
});
|