troxy-cli 1.21.1 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/bin/troxy.js +97 -22
- package/package.json +7 -4
- package/src/api.js +18 -1
- package/src/data/policy-spec.json +496 -0
- package/src/data/templates.json +1429 -0
- package/src/init.js +13 -2
- package/src/mcp-server.js +86 -1
- package/src/mcps.js +1 -0
- package/src/policies.js +351 -114
- package/src/policy-fields.js +107 -0
- package/src/secrets.js +1 -1
- package/src/settings.js +0 -3
- package/src/simulate.js +251 -0
- package/src/tests/mcp-checkpoints.test.js +79 -0
- package/src/tests/model-checkpoint.test.js +5 -1
- package/src/tests/policies-create.test.js +218 -0
- package/src/tests/policies-org.test.js +60 -0
- package/src/tests/simulate.test.js +127 -0
- package/src/tests/templates.test.js +46 -0
- package/src/tests/tool_detect.test.js +389 -0
- package/src/tool_detect.js +181 -0
- package/src/chat-budget.js +0 -73
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// Domain-aware condition validation for `policies create`, derived from the
|
|
2
|
+
// same vendored spec the dashboard checks itself against (src/data/policy-spec.json,
|
|
3
|
+
// see that file's _why and tools/export_policy_spec.py in troxy-tf-live for the
|
|
4
|
+
// full story). Before this existed, `policies create --field` only knew about
|
|
5
|
+
// three payment fields - amount, merchant_name, tx_per_day - because that was
|
|
6
|
+
// the whole product when the command was written. The other 44 fields across
|
|
7
|
+
// five more checkpoints (comms, access, destructive, model, secrets) were
|
|
8
|
+
// simply unreachable from the CLI, even though the backend and the dashboard
|
|
9
|
+
// both understood every one of them.
|
|
10
|
+
//
|
|
11
|
+
// Mirrors policies.py's _validate_field_op_and_value / _validate_condition_list
|
|
12
|
+
// so a bad condition is rejected here - offline, no network round-trip - instead
|
|
13
|
+
// of failing at the server with a message the user then has to map back onto
|
|
14
|
+
// what they typed.
|
|
15
|
+
import { readFileSync } from 'node:fs';
|
|
16
|
+
import { fileURLToPath } from 'node:url';
|
|
17
|
+
import { dirname, join } from 'node:path';
|
|
18
|
+
|
|
19
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
20
|
+
const SPEC = JSON.parse(readFileSync(join(__dirname, 'data', 'policy-spec.json'), 'utf8'));
|
|
21
|
+
|
|
22
|
+
export const DOMAIN_FIELDS = SPEC.domain_fields; // { domain: [field, ...] }
|
|
23
|
+
export const FIELD_SPECS = SPEC.field_specs; // { field: { ops:[...], kind, values?:[...] } }
|
|
24
|
+
export const DOMAINS = Object.keys(DOMAIN_FIELDS).sort();
|
|
25
|
+
|
|
26
|
+
// Friendly names for the canonical vocabulary. `access` means logins in the
|
|
27
|
+
// persisted `domain` column - a naming trap the dashboard has a standing
|
|
28
|
+
// comment about - so `login`/`signup` are accepted here rather than making
|
|
29
|
+
// every CLI user learn the column name to build a login policy.
|
|
30
|
+
const DOMAIN_ALIASES = {
|
|
31
|
+
payment: 'payment', payments: 'payment', spend: 'payment', spending: 'payment',
|
|
32
|
+
comms: 'comms', email: 'comms', messages: 'comms', messaging: 'comms', message: 'comms',
|
|
33
|
+
access: 'access', login: 'access', logins: 'access', signup: 'access', signin: 'access',
|
|
34
|
+
destructive: 'destructive', action: 'destructive', actions: 'destructive',
|
|
35
|
+
model: 'model', models: 'model', ai: 'model',
|
|
36
|
+
secrets: 'secrets', secret: 'secrets',
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export function normalizeDomain(input) {
|
|
40
|
+
if (!input) return null;
|
|
41
|
+
const key = String(input).trim().toLowerCase();
|
|
42
|
+
return DOMAIN_ALIASES[key] || null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Which fields belong to exactly one domain - used to infer --domain from
|
|
46
|
+
// --field when the user omits --domain, for backward compatibility with
|
|
47
|
+
// scripts written against the original payment-only command. A field shared
|
|
48
|
+
// by more than one domain (`destination` is both destructive and secrets) is
|
|
49
|
+
// deliberately excluded from inference: guessing wrong would silently attach
|
|
50
|
+
// a rule to the wrong checkpoint, which is the exact failure this file exists
|
|
51
|
+
// to prevent. Ambiguous fields require an explicit --domain.
|
|
52
|
+
const _domainsByField = {};
|
|
53
|
+
for (const [domain, fields] of Object.entries(DOMAIN_FIELDS)) {
|
|
54
|
+
for (const f of fields) (_domainsByField[f] ||= []).push(domain);
|
|
55
|
+
}
|
|
56
|
+
export function inferDomain(field) {
|
|
57
|
+
const ds = _domainsByField[field];
|
|
58
|
+
return ds && ds.length === 1 ? ds[0] : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function fieldsForDomain(domain) {
|
|
62
|
+
return DOMAIN_FIELDS[domain] || [];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Returns an error string, or null if the condition is valid. `field` may be
|
|
67
|
+
* omitted (a catch-all condition, matching every event in the domain) - only
|
|
68
|
+
* `domain` is mandatory whenever any condition is being validated at all.
|
|
69
|
+
*/
|
|
70
|
+
export function validateCondition(domain, field, operator, value, value2) {
|
|
71
|
+
if (!domain) return "domain is required (--domain, one of: " + DOMAINS.join(', ') + ")";
|
|
72
|
+
if (!DOMAINS.includes(domain)) return `domain must be one of: ${DOMAINS.join(', ')}`;
|
|
73
|
+
if (!field) return null;
|
|
74
|
+
|
|
75
|
+
const allowed = DOMAIN_FIELDS[domain] || [];
|
|
76
|
+
if (!allowed.includes(field)) {
|
|
77
|
+
return `'${field}' is not available on a '${domain}' policy (allowed: ${allowed.join(', ')})`;
|
|
78
|
+
}
|
|
79
|
+
const spec = FIELD_SPECS[field];
|
|
80
|
+
if (!spec) return `unknown field '${field}'`;
|
|
81
|
+
if (!operator) return `--operator is required with --field`;
|
|
82
|
+
if (!spec.ops.includes(operator)) {
|
|
83
|
+
return `'${operator}' does not apply to '${field}' - the engine has no way to compare it (allowed: ${spec.ops.join(', ')})`;
|
|
84
|
+
}
|
|
85
|
+
const text = value == null ? '' : String(value).trim();
|
|
86
|
+
if (!text) {
|
|
87
|
+
return `--value is required for '${field}' - an empty value matches everything or nothing depending on the operator`;
|
|
88
|
+
}
|
|
89
|
+
if (spec.kind === 'number') {
|
|
90
|
+
if (!isFinite(Number(text))) {
|
|
91
|
+
return `'${value}' is not a number, and '${field}' is read as 0 for anything that isn't - that would match far more than intended`;
|
|
92
|
+
}
|
|
93
|
+
if (operator === 'between') {
|
|
94
|
+
const t2 = value2 == null ? '' : String(value2).trim();
|
|
95
|
+
if (!t2 || !isFinite(Number(t2))) return `--value2 must be a number for 'between'`;
|
|
96
|
+
}
|
|
97
|
+
} else if (spec.kind === 'bool') {
|
|
98
|
+
if (!['true', 'false'].includes(text.toLowerCase())) {
|
|
99
|
+
return `'${field}' must be true or false, not '${value}'`;
|
|
100
|
+
}
|
|
101
|
+
} else if (spec.values && (operator === 'eq' || operator === 'neq')) {
|
|
102
|
+
if (!spec.values.includes(text.toLowerCase())) {
|
|
103
|
+
return `'${value}' is not one the '${field}' checkpoint can ever report (allowed: ${spec.values.join(', ')})`;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
package/src/secrets.js
CHANGED
|
@@ -4,7 +4,7 @@ import { requireJwt } from './auth.js';
|
|
|
4
4
|
import { table } from './print.js';
|
|
5
5
|
|
|
6
6
|
const HELP = {
|
|
7
|
-
list: ` troxy secrets list\n\n Lists your saved LLM provider keys
|
|
7
|
+
list: ` troxy secrets list\n\n Lists your saved LLM provider keys. Values are\n never shown, only a masked preview.\n`,
|
|
8
8
|
set: ` troxy secrets set --name <name> [--type api_key|text]\n\n Saves or updates an LLM provider key. You'll be prompted for the value\n so it never appears in your shell history. Login required.\n\n Options:\n --name Label for this key (e.g. "OpenAI", "Anthropic")\n --type api_key (default) or text\n\n Card secrets are dashboard-only; troxy secrets does not handle them.\n\n Example:\n troxy secrets set --name "OpenAI"\n`,
|
|
9
9
|
delete: ` troxy secrets delete --name <name>\n\n Deletes a saved key. Login required.\n`,
|
|
10
10
|
};
|
package/src/settings.js
CHANGED
|
@@ -30,9 +30,6 @@ export async function runSettings([sub, ...args], flags) {
|
|
|
30
30
|
On notify: ${s.notify_prefs.email_on_notify ? 'yes' : 'no'}
|
|
31
31
|
On allow: ${s.notify_prefs.email_on_allow ? 'yes' : 'no'}
|
|
32
32
|
Notify email: ${s.notify_prefs.notify_email || '(account email)'}
|
|
33
|
-
|
|
34
|
-
Chat budget action: ${s.chat_budget_action}
|
|
35
|
-
Chat budget limits: ${s.chat_budget_limits.length ? s.chat_budget_limits.map(b => `${b.currency} ${b.used}/${b.limit}`).join(', ') : 'none set, run troxy chat-budget'}
|
|
36
33
|
`);
|
|
37
34
|
break;
|
|
38
35
|
}
|
package/src/simulate.js
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
// Simulate a checkpoint call, the same way `troxy pay` simulates /evaluate.
|
|
2
|
+
// Before this file existed, `pay` was the only one - the other five
|
|
3
|
+
// checkpoints (email, login, destructive action, model, secrets) had no CLI
|
|
4
|
+
// equivalent at all, even though api.js already had wrappers for four of
|
|
5
|
+
// them (evaluateEmail/evaluateLogin/evaluateAction/evaluateModel) sitting
|
|
6
|
+
// unused, and evaluateSecrets was missing outright.
|
|
7
|
+
import { requireJwt } from './auth.js';
|
|
8
|
+
import { loadConfig } from './config.js';
|
|
9
|
+
import { api } from './api.js';
|
|
10
|
+
|
|
11
|
+
const ICON = { ALLOW: '✓', BLOCK: '✗', ESCALATE: '⏳', NOTIFY: '~' };
|
|
12
|
+
|
|
13
|
+
// The policy that answered lives in `result.policy` on /evaluate (payment)
|
|
14
|
+
// but is embedded in the `reason` sentence on every other checkpoint
|
|
15
|
+
// ("Matched policy 'X'."). Reading only `.policy`, the way an early version
|
|
16
|
+
// of this file did by copying `pay`'s formatting verbatim, silently showed
|
|
17
|
+
// "(default action)" on every one of these five commands even when a real
|
|
18
|
+
// policy had fired - the report would have been wrong from the day it shipped.
|
|
19
|
+
export function _policyName(result) {
|
|
20
|
+
if (result.policy) return result.policy;
|
|
21
|
+
const m = /[Mm]atched policy '([^']+)'/.exec(result.reason || '');
|
|
22
|
+
return m ? m[1] : null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function _requireApiKey() {
|
|
26
|
+
requireJwt();
|
|
27
|
+
const apiKey = loadConfig()?.apiKey || process.env.TROXY_API_KEY;
|
|
28
|
+
if (!apiKey) { console.error(' No API key. Run: troxy init --key txy-...\n'); process.exit(1); }
|
|
29
|
+
return apiKey;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function _agentName() {
|
|
33
|
+
return loadConfig()?.agentName || 'troxy-cli';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function _report(result) {
|
|
37
|
+
const icon = ICON[result.decision] || '?';
|
|
38
|
+
const policy = _policyName(result);
|
|
39
|
+
const suffix = policy ? ` ← "${policy}"` : result.reason ? ` (${result.reason})` : ' (default action)';
|
|
40
|
+
console.log(`\n ${icon} ${result.decision}${suffix}`);
|
|
41
|
+
if (result.audit_id) console.log(` audit: ${result.audit_id}`);
|
|
42
|
+
console.log();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const HELP = {
|
|
46
|
+
send: ` troxy send --recipient <address> [options]
|
|
47
|
+
|
|
48
|
+
Simulates a message evaluation (/evaluate/email) - the checkpoint an agent
|
|
49
|
+
calls before sending on any channel: email, WhatsApp, Telegram, Slack,
|
|
50
|
+
Teams, Discord, SMS. Use this to test your Messages policies. Login required.
|
|
51
|
+
|
|
52
|
+
Required:
|
|
53
|
+
--recipient <address> Who it's going to (address, phone, handle, or channel name)
|
|
54
|
+
|
|
55
|
+
Optional:
|
|
56
|
+
--recipients <a,b,c> Multiple recipients, comma-separated (adds to --recipient)
|
|
57
|
+
--subject <text> Subject line (ignored off email)
|
|
58
|
+
--body <text> Message body
|
|
59
|
+
--channel <name> email, whatsapp, telegram, slack, teams, discord, sms (default: email)
|
|
60
|
+
|
|
61
|
+
Examples:
|
|
62
|
+
troxy send --recipient jane@acme.com --subject "Q3 numbers" --body "attached"
|
|
63
|
+
troxy send --recipient jane@gmail.com --body "the password is hunter2"
|
|
64
|
+
troxy send --recipient "#eng" --channel slack --body "deploying now"
|
|
65
|
+
`,
|
|
66
|
+
signin: ` troxy signin --site <domain> [--type login|signup]
|
|
67
|
+
|
|
68
|
+
Simulates a login/signup evaluation (/evaluate/login). Use this to test your
|
|
69
|
+
Access & privacy policies. Login required.
|
|
70
|
+
|
|
71
|
+
Required:
|
|
72
|
+
--site <domain> Site the agent is signing into (e.g. "coinbase.com")
|
|
73
|
+
|
|
74
|
+
Optional:
|
|
75
|
+
--type <type> login or signup (default: login)
|
|
76
|
+
|
|
77
|
+
Examples:
|
|
78
|
+
troxy signin --site coinbase.com --type signup
|
|
79
|
+
troxy signin --site github.com
|
|
80
|
+
`,
|
|
81
|
+
action: ` troxy action --verb <verb> --resource <what> [options]
|
|
82
|
+
|
|
83
|
+
Simulates a consequential-action evaluation (/evaluate/action) - deletes,
|
|
84
|
+
drops, overwrites, force-pushes, grants, exports, and other actions with
|
|
85
|
+
real consequences. Use this to test your Destructive/Access policies. Login required.
|
|
86
|
+
|
|
87
|
+
Required:
|
|
88
|
+
--verb <verb> delete, drop, destroy, overwrite, force_push, share,
|
|
89
|
+
grant_access, add_user, rotate, modify, export
|
|
90
|
+
--resource <text> What it acts on (e.g. "prod users table")
|
|
91
|
+
|
|
92
|
+
Optional:
|
|
93
|
+
--destination <who> Who can reach it afterwards: an address, a domain, a
|
|
94
|
+
team name, or "public". Omit for actions that don't
|
|
95
|
+
change who can see anything (e.g. storing your own file).
|
|
96
|
+
--items <n> How many things this affects (default: 1)
|
|
97
|
+
--reversible <bool> true or false (default: false)
|
|
98
|
+
|
|
99
|
+
Examples:
|
|
100
|
+
troxy action --verb delete --resource "prod users table" --items 500 --reversible false
|
|
101
|
+
troxy action --verb share --resource "Q3 revenue doc" --destination public
|
|
102
|
+
troxy action --verb modify --resource "billing settings"
|
|
103
|
+
`,
|
|
104
|
+
'model-call': ` troxy model-call --model <id> --provider <name> [options]
|
|
105
|
+
|
|
106
|
+
Simulates a model-usage evaluation (/evaluate/model) - what an agent calls
|
|
107
|
+
before starting a run on a given model. Use this to test your AI & models
|
|
108
|
+
policies. Login required.
|
|
109
|
+
|
|
110
|
+
Required:
|
|
111
|
+
--model <id> Model id, e.g. claude-opus-5
|
|
112
|
+
--provider <name> anthropic, openai, google, or wherever it runs
|
|
113
|
+
|
|
114
|
+
Optional:
|
|
115
|
+
--cost <n> Your best estimate of this run's cost in USD (default: 0)
|
|
116
|
+
--tokens <n> Tokens you expect to use
|
|
117
|
+
--effort <level> low, medium, or high (reasoning/thinking intensity)
|
|
118
|
+
|
|
119
|
+
Examples:
|
|
120
|
+
troxy model-call --model claude-opus-5 --provider anthropic --cost 4.50 --tokens 900000
|
|
121
|
+
troxy model-call --model gpt-4o --provider openai --cost 0.02 --tokens 500
|
|
122
|
+
`,
|
|
123
|
+
scan: ` troxy scan --content <text> [options]
|
|
124
|
+
|
|
125
|
+
Simulates a sensitive-data evaluation (/evaluate/secrets) - the checkpoint
|
|
126
|
+
an agent calls before sending, writing, or publishing content that might
|
|
127
|
+
contain a secret. Content is scanned server-side; you don't pre-classify it.
|
|
128
|
+
Use this to test your Secrets & credentials policies. Login required.
|
|
129
|
+
|
|
130
|
+
Required:
|
|
131
|
+
--content <text> The text/code/data about to be sent or written
|
|
132
|
+
|
|
133
|
+
Optional:
|
|
134
|
+
--destination <where> A person, repo, channel, service, or "public" (default: public)
|
|
135
|
+
--do <action> send, push, upload, write, or share (default: send)
|
|
136
|
+
--type <type> code, message, file, config, log, or other (default: message)
|
|
137
|
+
|
|
138
|
+
Examples:
|
|
139
|
+
troxy scan --content "AWS_SECRET_ACCESS_KEY=wJalr...EXAMPLEKEY" --destination public --do push --type code
|
|
140
|
+
troxy scan --content "hey the password is hunter2" --do send --type message
|
|
141
|
+
`,
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
export function printSimHelp(command) {
|
|
145
|
+
console.log('\n' + HELP[command]);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Body builders are pure and exported separately from the run* functions
|
|
149
|
+
// below, so field-name mistakes - the exact class of bug this whole session
|
|
150
|
+
// has been about, a request shaped slightly wrong that saves/sends fine and
|
|
151
|
+
// then never matches a policy - are testable without a live login session.
|
|
152
|
+
// _requireApiKey() exits the process if there's no session, which would make
|
|
153
|
+
// every one of these untestable in CI if the body-building logic lived
|
|
154
|
+
// inside the same function.
|
|
155
|
+
|
|
156
|
+
export function _sendBody(flags) {
|
|
157
|
+
if (!flags.recipient && !flags.recipients) { console.error(' --recipient is required\n'); process.exit(1); }
|
|
158
|
+
const recipients = [
|
|
159
|
+
...(flags.recipient ? [flags.recipient] : []),
|
|
160
|
+
...(flags.recipients ? String(flags.recipients).split(',').map(s => s.trim()).filter(Boolean) : []),
|
|
161
|
+
];
|
|
162
|
+
return {
|
|
163
|
+
recipient: recipients[0],
|
|
164
|
+
recipients,
|
|
165
|
+
subject: flags.subject || '',
|
|
166
|
+
body: flags.body || '',
|
|
167
|
+
channel: flags.channel || 'email',
|
|
168
|
+
agent: _agentName(),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function _signinBody(flags) {
|
|
173
|
+
if (!flags.site) { console.error(' --site is required\n'); process.exit(1); }
|
|
174
|
+
const type = (flags.type || 'login').toLowerCase();
|
|
175
|
+
if (!['login', 'signup'].includes(type)) { console.error(' --type must be login or signup\n'); process.exit(1); }
|
|
176
|
+
return { site: flags.site, login_action: type, agent: _agentName() };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function _actionBody(flags) {
|
|
180
|
+
if (!flags.verb) { console.error(' --verb is required\n'); process.exit(1); }
|
|
181
|
+
if (!flags.resource) { console.error(' --resource is required\n'); process.exit(1); }
|
|
182
|
+
const items = flags.items != null ? parseInt(flags.items, 10) : 1;
|
|
183
|
+
if (isNaN(items)) { console.error(' --items must be a number\n'); process.exit(1); }
|
|
184
|
+
const reversible = flags.reversible != null ? String(flags.reversible).toLowerCase() === 'true' : false;
|
|
185
|
+
const body = {
|
|
186
|
+
action_verb: flags.verb,
|
|
187
|
+
resource: flags.resource,
|
|
188
|
+
item_count: items,
|
|
189
|
+
reversible,
|
|
190
|
+
agent: _agentName(),
|
|
191
|
+
};
|
|
192
|
+
if (flags.destination) body.destination = flags.destination;
|
|
193
|
+
return body;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function _modelCallBody(flags) {
|
|
197
|
+
if (!flags.model) { console.error(' --model is required\n'); process.exit(1); }
|
|
198
|
+
if (!flags.provider) { console.error(' --provider is required\n'); process.exit(1); }
|
|
199
|
+
const cost = flags.cost != null ? parseFloat(flags.cost) : 0;
|
|
200
|
+
if (isNaN(cost)) { console.error(' --cost must be a number\n'); process.exit(1); }
|
|
201
|
+
const body = { model: flags.model, provider: flags.provider, estimated_cost: cost, agent: _agentName() };
|
|
202
|
+
if (flags.tokens != null) {
|
|
203
|
+
const tokens = parseInt(flags.tokens, 10);
|
|
204
|
+
if (isNaN(tokens)) { console.error(' --tokens must be a number\n'); process.exit(1); }
|
|
205
|
+
body.tokens = tokens;
|
|
206
|
+
}
|
|
207
|
+
if (flags.effort) body.effort = flags.effort;
|
|
208
|
+
return body;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function _scanBody(flags) {
|
|
212
|
+
if (!flags.content) { console.error(' --content is required\n'); process.exit(1); }
|
|
213
|
+
return {
|
|
214
|
+
content: flags.content,
|
|
215
|
+
destination: flags.destination || 'public',
|
|
216
|
+
action: flags.do || 'send',
|
|
217
|
+
content_type: flags.type || 'message',
|
|
218
|
+
agent: _agentName(),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export async function runSend(flags) {
|
|
223
|
+
const apiKey = _requireApiKey();
|
|
224
|
+
_report(await api.evaluateEmail(_sendBody(flags), apiKey));
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export async function runSignin(flags) {
|
|
228
|
+
const apiKey = _requireApiKey();
|
|
229
|
+
_report(await api.evaluateLogin(_signinBody(flags), apiKey));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export async function runAction(flags) {
|
|
233
|
+
const apiKey = _requireApiKey();
|
|
234
|
+
_report(await api.evaluateAction(_actionBody(flags), apiKey));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export async function runModelCall(flags) {
|
|
238
|
+
const apiKey = _requireApiKey();
|
|
239
|
+
_report(await api.evaluateModel(_modelCallBody(flags), apiKey));
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export async function runScan(flags) {
|
|
243
|
+
const apiKey = _requireApiKey();
|
|
244
|
+
const result = await api.evaluateSecrets(_scanBody(flags), apiKey);
|
|
245
|
+
_report(result);
|
|
246
|
+
if (result.detection_count) {
|
|
247
|
+
console.log(` ${result.detection_count} thing(s) detected, highest severity: ${result.highest_severity || 'n/a'}`);
|
|
248
|
+
(result.detections || []).forEach(d => console.log(` - ${d.type} (${d.severity}): ${d.preview}`));
|
|
249
|
+
console.log();
|
|
250
|
+
}
|
|
251
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Every checkpoint the product has needs an MCP tool, or an MCP-connected
|
|
2
|
+
// agent - which is how most users connect - simply cannot reach it. Not "the
|
|
3
|
+
// agent chose not to call it": there is nothing to call.
|
|
4
|
+
//
|
|
5
|
+
// Found live, 2026-08-15: asked a connected agent to leak an AWS key and to
|
|
6
|
+
// bulk-delete prod rows, in an ordinary conversation, no mention of Troxy.
|
|
7
|
+
// The delete got refused (evaluate_action exists). The key-leak request was
|
|
8
|
+
// just carried out - not because the agent ignored a guardrail, but because
|
|
9
|
+
// evaluate_secrets had no MCP tool at all. Every other checkpoint
|
|
10
|
+
// (payment/email/login/action/model) had one; secrets did not. The backend's
|
|
11
|
+
// own other_checkpoints discovery (every /evaluate* response advertises the
|
|
12
|
+
// other five) did mention it, but as a raw HTTP URL - useless to an agent
|
|
13
|
+
// whose only way to reach Troxy is through the tools this file defines.
|
|
14
|
+
//
|
|
15
|
+
// model-checkpoint.test.js already covers this exact wiring for model/
|
|
16
|
+
// report_model_usage specifically, following a near-identical 2026-08-06
|
|
17
|
+
// incident (this is the second time a checkpoint has shipped on the API and
|
|
18
|
+
// dashboard without its MCP tool). This file generalizes the check across
|
|
19
|
+
// every checkpoint so a sixth one added later can't repeat it.
|
|
20
|
+
import { test } from 'node:test';
|
|
21
|
+
import assert from 'node:assert';
|
|
22
|
+
import { readFileSync } from 'node:fs';
|
|
23
|
+
import { fileURLToPath } from 'node:url';
|
|
24
|
+
import { dirname, join } from 'node:path';
|
|
25
|
+
|
|
26
|
+
const src = readFileSync(
|
|
27
|
+
join(dirname(fileURLToPath(import.meta.url)), '..', 'mcp-server.js'),
|
|
28
|
+
'utf8',
|
|
29
|
+
);
|
|
30
|
+
const apiSrc = readFileSync(
|
|
31
|
+
join(dirname(fileURLToPath(import.meta.url)), '..', 'api.js'),
|
|
32
|
+
'utf8',
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
// One MCP tool per checkpoint, and the /evaluate* path each one must reach.
|
|
36
|
+
// Deliberately a literal, not derived from mcp-server.js itself - a test
|
|
37
|
+
// that reads its expectation from the code under test cannot fail the way
|
|
38
|
+
// this one needs to, which is exactly how secrets went unnoticed despite
|
|
39
|
+
// model-checkpoint.test.js already existing as a precedent for this check.
|
|
40
|
+
const CHECKPOINTS = {
|
|
41
|
+
evaluate_payment: '/evaluate',
|
|
42
|
+
evaluate_email: '/evaluate/email',
|
|
43
|
+
evaluate_login: '/evaluate/login',
|
|
44
|
+
evaluate_action: '/evaluate/action',
|
|
45
|
+
evaluate_model: '/evaluate/model',
|
|
46
|
+
evaluate_secrets: '/evaluate/secrets',
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
test('every checkpoint has an MCP tool with a handler', () => {
|
|
50
|
+
for (const name of Object.keys(CHECKPOINTS)) {
|
|
51
|
+
assert.ok(
|
|
52
|
+
src.includes(`name: '${name}'`),
|
|
53
|
+
`${name} is not in the MCP tool list - an MCP-connected agent has no way to call it at all`,
|
|
54
|
+
);
|
|
55
|
+
assert.ok(
|
|
56
|
+
src.includes(`toolName === '${name}'`) || (name === 'evaluate_payment' && src.includes("toolName !== 'evaluate_payment'")),
|
|
57
|
+
`${name} is advertised but has no handler, so calling it does nothing`,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('every checkpoint tool is wired to its real endpoint in the api client', () => {
|
|
63
|
+
for (const [name, path] of Object.entries(CHECKPOINTS)) {
|
|
64
|
+
assert.ok(
|
|
65
|
+
apiSrc.includes(`'${path}'`),
|
|
66
|
+
`${name} has no api.js wrapper for ${path} - the MCP tool would exist with nothing behind it`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('evaluate_secrets is exported from mcp-server.js\'s own api bindings, not just referenced', () => {
|
|
72
|
+
// The other tools each get a one-line local binding
|
|
73
|
+
// (`const evaluateAction = (body, apiKey) => api.evaluateAction(...)`)
|
|
74
|
+
// before the tool list. secrets needs the same, or the handler below
|
|
75
|
+
// throws a ReferenceError the first time an agent actually calls it -
|
|
76
|
+
// which the two tests above cannot see, since they only check that the
|
|
77
|
+
// NAME 'evaluateSecrets' appears somewhere, not that it resolves.
|
|
78
|
+
assert.match(src, /const evaluateSecrets\s*=\s*\(body, apiKey\)\s*=>\s*api\.evaluateSecrets\(body, apiKey\)/);
|
|
79
|
+
});
|
|
@@ -242,6 +242,10 @@ test('no new tool was introduced for the chat path', () => {
|
|
|
242
242
|
const toolsSeg = src.slice(src.indexOf('tools: ['), src.indexOf('server.setRequestHandler(CallToolRequestSchema'));
|
|
243
243
|
const toolNames = [...toolsSeg.matchAll(/name: '([a-z_]+)'/g)].map(m => m[1]);
|
|
244
244
|
const unique = new Set(toolNames);
|
|
245
|
-
|
|
245
|
+
// 9, not this test's original 8: evaluate_secrets was added afterward
|
|
246
|
+
// (2026-08-15, mcp-checkpoints.test.js) to close a real gap, not to give
|
|
247
|
+
// chat its own tool - the assertion this test cares about is still that no
|
|
248
|
+
// *tenth*, chat-specific tool exists.
|
|
249
|
+
assert.strictEqual(unique.size, 9, 'a tool was added or removed for this change');
|
|
246
250
|
assert.ok(unique.has('evaluate_model') && unique.has('report_model_usage'));
|
|
247
251
|
});
|