troxy-cli 1.21.1 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/troxy.js +92 -12
- package/package.json +7 -4
- package/src/api.js +18 -0
- package/src/data/policy-spec.json +496 -0
- package/src/data/templates.json +1429 -0
- package/src/init.js +11 -0
- package/src/mcp-server.js +81 -0
- package/src/policies.js +341 -46
- package/src/policy-fields.js +107 -0
- package/src/simulate.js +251 -0
- package/src/tests/mcp-checkpoints.test.js +79 -0
- package/src/tests/model-checkpoint.test.js +5 -1
- package/src/tests/policies-create.test.js +218 -0
- package/src/tests/policies-org.test.js +60 -0
- package/src/tests/simulate.test.js +127 -0
- package/src/tests/templates.test.js +46 -0
- package/src/tool_detect.js +179 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// Domain-aware condition validation for `policies create`, derived from the
|
|
2
|
+
// same vendored spec the dashboard checks itself against (src/data/policy-spec.json,
|
|
3
|
+
// see that file's _why and tools/export_policy_spec.py in troxy-tf-live for the
|
|
4
|
+
// full story). Before this existed, `policies create --field` only knew about
|
|
5
|
+
// three payment fields - amount, merchant_name, tx_per_day - because that was
|
|
6
|
+
// the whole product when the command was written. The other 44 fields across
|
|
7
|
+
// five more checkpoints (comms, access, destructive, model, secrets) were
|
|
8
|
+
// simply unreachable from the CLI, even though the backend and the dashboard
|
|
9
|
+
// both understood every one of them.
|
|
10
|
+
//
|
|
11
|
+
// Mirrors policies.py's _validate_field_op_and_value / _validate_condition_list
|
|
12
|
+
// so a bad condition is rejected here - offline, no network round-trip - instead
|
|
13
|
+
// of failing at the server with a message the user then has to map back onto
|
|
14
|
+
// what they typed.
|
|
15
|
+
import { readFileSync } from 'node:fs';
|
|
16
|
+
import { fileURLToPath } from 'node:url';
|
|
17
|
+
import { dirname, join } from 'node:path';
|
|
18
|
+
|
|
19
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
20
|
+
const SPEC = JSON.parse(readFileSync(join(__dirname, 'data', 'policy-spec.json'), 'utf8'));
|
|
21
|
+
|
|
22
|
+
export const DOMAIN_FIELDS = SPEC.domain_fields; // { domain: [field, ...] }
|
|
23
|
+
export const FIELD_SPECS = SPEC.field_specs; // { field: { ops:[...], kind, values?:[...] } }
|
|
24
|
+
export const DOMAINS = Object.keys(DOMAIN_FIELDS).sort();
|
|
25
|
+
|
|
26
|
+
// Friendly names for the canonical vocabulary. `access` means logins in the
|
|
27
|
+
// persisted `domain` column - a naming trap the dashboard has a standing
|
|
28
|
+
// comment about - so `login`/`signup` are accepted here rather than making
|
|
29
|
+
// every CLI user learn the column name to build a login policy.
|
|
30
|
+
const DOMAIN_ALIASES = {
|
|
31
|
+
payment: 'payment', payments: 'payment', spend: 'payment', spending: 'payment',
|
|
32
|
+
comms: 'comms', email: 'comms', messages: 'comms', messaging: 'comms', message: 'comms',
|
|
33
|
+
access: 'access', login: 'access', logins: 'access', signup: 'access', signin: 'access',
|
|
34
|
+
destructive: 'destructive', action: 'destructive', actions: 'destructive',
|
|
35
|
+
model: 'model', models: 'model', ai: 'model',
|
|
36
|
+
secrets: 'secrets', secret: 'secrets',
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export function normalizeDomain(input) {
|
|
40
|
+
if (!input) return null;
|
|
41
|
+
const key = String(input).trim().toLowerCase();
|
|
42
|
+
return DOMAIN_ALIASES[key] || null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Which fields belong to exactly one domain - used to infer --domain from
|
|
46
|
+
// --field when the user omits --domain, for backward compatibility with
|
|
47
|
+
// scripts written against the original payment-only command. A field shared
|
|
48
|
+
// by more than one domain (`destination` is both destructive and secrets) is
|
|
49
|
+
// deliberately excluded from inference: guessing wrong would silently attach
|
|
50
|
+
// a rule to the wrong checkpoint, which is the exact failure this file exists
|
|
51
|
+
// to prevent. Ambiguous fields require an explicit --domain.
|
|
52
|
+
const _domainsByField = {};
|
|
53
|
+
for (const [domain, fields] of Object.entries(DOMAIN_FIELDS)) {
|
|
54
|
+
for (const f of fields) (_domainsByField[f] ||= []).push(domain);
|
|
55
|
+
}
|
|
56
|
+
export function inferDomain(field) {
|
|
57
|
+
const ds = _domainsByField[field];
|
|
58
|
+
return ds && ds.length === 1 ? ds[0] : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function fieldsForDomain(domain) {
|
|
62
|
+
return DOMAIN_FIELDS[domain] || [];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Returns an error string, or null if the condition is valid. `field` may be
|
|
67
|
+
* omitted (a catch-all condition, matching every event in the domain) - only
|
|
68
|
+
* `domain` is mandatory whenever any condition is being validated at all.
|
|
69
|
+
*/
|
|
70
|
+
export function validateCondition(domain, field, operator, value, value2) {
|
|
71
|
+
if (!domain) return "domain is required (--domain, one of: " + DOMAINS.join(', ') + ")";
|
|
72
|
+
if (!DOMAINS.includes(domain)) return `domain must be one of: ${DOMAINS.join(', ')}`;
|
|
73
|
+
if (!field) return null;
|
|
74
|
+
|
|
75
|
+
const allowed = DOMAIN_FIELDS[domain] || [];
|
|
76
|
+
if (!allowed.includes(field)) {
|
|
77
|
+
return `'${field}' is not available on a '${domain}' policy (allowed: ${allowed.join(', ')})`;
|
|
78
|
+
}
|
|
79
|
+
const spec = FIELD_SPECS[field];
|
|
80
|
+
if (!spec) return `unknown field '${field}'`;
|
|
81
|
+
if (!operator) return `--operator is required with --field`;
|
|
82
|
+
if (!spec.ops.includes(operator)) {
|
|
83
|
+
return `'${operator}' does not apply to '${field}' - the engine has no way to compare it (allowed: ${spec.ops.join(', ')})`;
|
|
84
|
+
}
|
|
85
|
+
const text = value == null ? '' : String(value).trim();
|
|
86
|
+
if (!text) {
|
|
87
|
+
return `--value is required for '${field}' - an empty value matches everything or nothing depending on the operator`;
|
|
88
|
+
}
|
|
89
|
+
if (spec.kind === 'number') {
|
|
90
|
+
if (!isFinite(Number(text))) {
|
|
91
|
+
return `'${value}' is not a number, and '${field}' is read as 0 for anything that isn't - that would match far more than intended`;
|
|
92
|
+
}
|
|
93
|
+
if (operator === 'between') {
|
|
94
|
+
const t2 = value2 == null ? '' : String(value2).trim();
|
|
95
|
+
if (!t2 || !isFinite(Number(t2))) return `--value2 must be a number for 'between'`;
|
|
96
|
+
}
|
|
97
|
+
} else if (spec.kind === 'bool') {
|
|
98
|
+
if (!['true', 'false'].includes(text.toLowerCase())) {
|
|
99
|
+
return `'${field}' must be true or false, not '${value}'`;
|
|
100
|
+
}
|
|
101
|
+
} else if (spec.values && (operator === 'eq' || operator === 'neq')) {
|
|
102
|
+
if (!spec.values.includes(text.toLowerCase())) {
|
|
103
|
+
return `'${value}' is not one the '${field}' checkpoint can ever report (allowed: ${spec.values.join(', ')})`;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
package/src/simulate.js
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
// Simulate a checkpoint call, the same way `troxy pay` simulates /evaluate.
|
|
2
|
+
// Before this file existed, `pay` was the only one - the other five
|
|
3
|
+
// checkpoints (email, login, destructive action, model, secrets) had no CLI
|
|
4
|
+
// equivalent at all, even though api.js already had wrappers for four of
|
|
5
|
+
// them (evaluateEmail/evaluateLogin/evaluateAction/evaluateModel) sitting
|
|
6
|
+
// unused, and evaluateSecrets was missing outright.
|
|
7
|
+
import { requireJwt } from './auth.js';
|
|
8
|
+
import { loadConfig } from './config.js';
|
|
9
|
+
import { api } from './api.js';
|
|
10
|
+
|
|
11
|
+
const ICON = { ALLOW: '✓', BLOCK: '✗', ESCALATE: '⏳', NOTIFY: '~' };
|
|
12
|
+
|
|
13
|
+
// The policy that answered lives in `result.policy` on /evaluate (payment)
|
|
14
|
+
// but is embedded in the `reason` sentence on every other checkpoint
|
|
15
|
+
// ("Matched policy 'X'."). Reading only `.policy`, the way an early version
|
|
16
|
+
// of this file did by copying `pay`'s formatting verbatim, silently showed
|
|
17
|
+
// "(default action)" on every one of these five commands even when a real
|
|
18
|
+
// policy had fired - the report would have been wrong from the day it shipped.
|
|
19
|
+
export function _policyName(result) {
|
|
20
|
+
if (result.policy) return result.policy;
|
|
21
|
+
const m = /[Mm]atched policy '([^']+)'/.exec(result.reason || '');
|
|
22
|
+
return m ? m[1] : null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function _requireApiKey() {
|
|
26
|
+
requireJwt();
|
|
27
|
+
const apiKey = loadConfig()?.apiKey || process.env.TROXY_API_KEY;
|
|
28
|
+
if (!apiKey) { console.error(' No API key. Run: troxy init --key txy-...\n'); process.exit(1); }
|
|
29
|
+
return apiKey;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function _agentName() {
|
|
33
|
+
return loadConfig()?.agentName || 'troxy-cli';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function _report(result) {
|
|
37
|
+
const icon = ICON[result.decision] || '?';
|
|
38
|
+
const policy = _policyName(result);
|
|
39
|
+
const suffix = policy ? ` ← "${policy}"` : result.reason ? ` (${result.reason})` : ' (default action)';
|
|
40
|
+
console.log(`\n ${icon} ${result.decision}${suffix}`);
|
|
41
|
+
if (result.audit_id) console.log(` audit: ${result.audit_id}`);
|
|
42
|
+
console.log();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const HELP = {
|
|
46
|
+
send: ` troxy send --recipient <address> [options]
|
|
47
|
+
|
|
48
|
+
Simulates a message evaluation (/evaluate/email) - the checkpoint an agent
|
|
49
|
+
calls before sending on any channel: email, WhatsApp, Telegram, Slack,
|
|
50
|
+
Teams, Discord, SMS. Use this to test your Messages policies. Login required.
|
|
51
|
+
|
|
52
|
+
Required:
|
|
53
|
+
--recipient <address> Who it's going to (address, phone, handle, or channel name)
|
|
54
|
+
|
|
55
|
+
Optional:
|
|
56
|
+
--recipients <a,b,c> Multiple recipients, comma-separated (adds to --recipient)
|
|
57
|
+
--subject <text> Subject line (ignored off email)
|
|
58
|
+
--body <text> Message body
|
|
59
|
+
--channel <name> email, whatsapp, telegram, slack, teams, discord, sms (default: email)
|
|
60
|
+
|
|
61
|
+
Examples:
|
|
62
|
+
troxy send --recipient jane@acme.com --subject "Q3 numbers" --body "attached"
|
|
63
|
+
troxy send --recipient jane@gmail.com --body "the password is hunter2"
|
|
64
|
+
troxy send --recipient "#eng" --channel slack --body "deploying now"
|
|
65
|
+
`,
|
|
66
|
+
signin: ` troxy signin --site <domain> [--type login|signup]
|
|
67
|
+
|
|
68
|
+
Simulates a login/signup evaluation (/evaluate/login). Use this to test your
|
|
69
|
+
Access & privacy policies. Login required.
|
|
70
|
+
|
|
71
|
+
Required:
|
|
72
|
+
--site <domain> Site the agent is signing into (e.g. "coinbase.com")
|
|
73
|
+
|
|
74
|
+
Optional:
|
|
75
|
+
--type <type> login or signup (default: login)
|
|
76
|
+
|
|
77
|
+
Examples:
|
|
78
|
+
troxy signin --site coinbase.com --type signup
|
|
79
|
+
troxy signin --site github.com
|
|
80
|
+
`,
|
|
81
|
+
action: ` troxy action --verb <verb> --resource <what> [options]
|
|
82
|
+
|
|
83
|
+
Simulates a consequential-action evaluation (/evaluate/action) - deletes,
|
|
84
|
+
drops, overwrites, force-pushes, grants, exports, and other actions with
|
|
85
|
+
real consequences. Use this to test your Destructive/Access policies. Login required.
|
|
86
|
+
|
|
87
|
+
Required:
|
|
88
|
+
--verb <verb> delete, drop, destroy, overwrite, force_push, share,
|
|
89
|
+
grant_access, add_user, rotate, modify, export
|
|
90
|
+
--resource <text> What it acts on (e.g. "prod users table")
|
|
91
|
+
|
|
92
|
+
Optional:
|
|
93
|
+
--destination <who> Who can reach it afterwards: an address, a domain, a
|
|
94
|
+
team name, or "public". Omit for actions that don't
|
|
95
|
+
change who can see anything (e.g. storing your own file).
|
|
96
|
+
--items <n> How many things this affects (default: 1)
|
|
97
|
+
--reversible <bool> true or false (default: false)
|
|
98
|
+
|
|
99
|
+
Examples:
|
|
100
|
+
troxy action --verb delete --resource "prod users table" --items 500 --reversible false
|
|
101
|
+
troxy action --verb share --resource "Q3 revenue doc" --destination public
|
|
102
|
+
troxy action --verb modify --resource "billing settings"
|
|
103
|
+
`,
|
|
104
|
+
'model-call': ` troxy model-call --model <id> --provider <name> [options]
|
|
105
|
+
|
|
106
|
+
Simulates a model-usage evaluation (/evaluate/model) - what an agent calls
|
|
107
|
+
before starting a run on a given model. Use this to test your AI & models
|
|
108
|
+
policies. Login required.
|
|
109
|
+
|
|
110
|
+
Required:
|
|
111
|
+
--model <id> Model id, e.g. claude-opus-5
|
|
112
|
+
--provider <name> anthropic, openai, google, or wherever it runs
|
|
113
|
+
|
|
114
|
+
Optional:
|
|
115
|
+
--cost <n> Your best estimate of this run's cost in USD (default: 0)
|
|
116
|
+
--tokens <n> Tokens you expect to use
|
|
117
|
+
--effort <level> low, medium, or high (reasoning/thinking intensity)
|
|
118
|
+
|
|
119
|
+
Examples:
|
|
120
|
+
troxy model-call --model claude-opus-5 --provider anthropic --cost 4.50 --tokens 900000
|
|
121
|
+
troxy model-call --model gpt-4o --provider openai --cost 0.02 --tokens 500
|
|
122
|
+
`,
|
|
123
|
+
scan: ` troxy scan --content <text> [options]
|
|
124
|
+
|
|
125
|
+
Simulates a sensitive-data evaluation (/evaluate/secrets) - the checkpoint
|
|
126
|
+
an agent calls before sending, writing, or publishing content that might
|
|
127
|
+
contain a secret. Content is scanned server-side; you don't pre-classify it.
|
|
128
|
+
Use this to test your Secrets & credentials policies. Login required.
|
|
129
|
+
|
|
130
|
+
Required:
|
|
131
|
+
--content <text> The text/code/data about to be sent or written
|
|
132
|
+
|
|
133
|
+
Optional:
|
|
134
|
+
--destination <where> A person, repo, channel, service, or "public" (default: public)
|
|
135
|
+
--do <action> send, push, upload, write, or share (default: send)
|
|
136
|
+
--type <type> code, message, file, config, log, or other (default: message)
|
|
137
|
+
|
|
138
|
+
Examples:
|
|
139
|
+
troxy scan --content "AWS_SECRET_ACCESS_KEY=wJalr...EXAMPLEKEY" --destination public --do push --type code
|
|
140
|
+
troxy scan --content "hey the password is hunter2" --do send --type message
|
|
141
|
+
`,
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
export function printSimHelp(command) {
|
|
145
|
+
console.log('\n' + HELP[command]);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Body builders are pure and exported separately from the run* functions
|
|
149
|
+
// below, so field-name mistakes - the exact class of bug this whole session
|
|
150
|
+
// has been about, a request shaped slightly wrong that saves/sends fine and
|
|
151
|
+
// then never matches a policy - are testable without a live login session.
|
|
152
|
+
// _requireApiKey() exits the process if there's no session, which would make
|
|
153
|
+
// every one of these untestable in CI if the body-building logic lived
|
|
154
|
+
// inside the same function.
|
|
155
|
+
|
|
156
|
+
export function _sendBody(flags) {
|
|
157
|
+
if (!flags.recipient && !flags.recipients) { console.error(' --recipient is required\n'); process.exit(1); }
|
|
158
|
+
const recipients = [
|
|
159
|
+
...(flags.recipient ? [flags.recipient] : []),
|
|
160
|
+
...(flags.recipients ? String(flags.recipients).split(',').map(s => s.trim()).filter(Boolean) : []),
|
|
161
|
+
];
|
|
162
|
+
return {
|
|
163
|
+
recipient: recipients[0],
|
|
164
|
+
recipients,
|
|
165
|
+
subject: flags.subject || '',
|
|
166
|
+
body: flags.body || '',
|
|
167
|
+
channel: flags.channel || 'email',
|
|
168
|
+
agent: _agentName(),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function _signinBody(flags) {
|
|
173
|
+
if (!flags.site) { console.error(' --site is required\n'); process.exit(1); }
|
|
174
|
+
const type = (flags.type || 'login').toLowerCase();
|
|
175
|
+
if (!['login', 'signup'].includes(type)) { console.error(' --type must be login or signup\n'); process.exit(1); }
|
|
176
|
+
return { site: flags.site, login_action: type, agent: _agentName() };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function _actionBody(flags) {
|
|
180
|
+
if (!flags.verb) { console.error(' --verb is required\n'); process.exit(1); }
|
|
181
|
+
if (!flags.resource) { console.error(' --resource is required\n'); process.exit(1); }
|
|
182
|
+
const items = flags.items != null ? parseInt(flags.items, 10) : 1;
|
|
183
|
+
if (isNaN(items)) { console.error(' --items must be a number\n'); process.exit(1); }
|
|
184
|
+
const reversible = flags.reversible != null ? String(flags.reversible).toLowerCase() === 'true' : false;
|
|
185
|
+
const body = {
|
|
186
|
+
action_verb: flags.verb,
|
|
187
|
+
resource: flags.resource,
|
|
188
|
+
item_count: items,
|
|
189
|
+
reversible,
|
|
190
|
+
agent: _agentName(),
|
|
191
|
+
};
|
|
192
|
+
if (flags.destination) body.destination = flags.destination;
|
|
193
|
+
return body;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function _modelCallBody(flags) {
|
|
197
|
+
if (!flags.model) { console.error(' --model is required\n'); process.exit(1); }
|
|
198
|
+
if (!flags.provider) { console.error(' --provider is required\n'); process.exit(1); }
|
|
199
|
+
const cost = flags.cost != null ? parseFloat(flags.cost) : 0;
|
|
200
|
+
if (isNaN(cost)) { console.error(' --cost must be a number\n'); process.exit(1); }
|
|
201
|
+
const body = { model: flags.model, provider: flags.provider, estimated_cost: cost, agent: _agentName() };
|
|
202
|
+
if (flags.tokens != null) {
|
|
203
|
+
const tokens = parseInt(flags.tokens, 10);
|
|
204
|
+
if (isNaN(tokens)) { console.error(' --tokens must be a number\n'); process.exit(1); }
|
|
205
|
+
body.tokens = tokens;
|
|
206
|
+
}
|
|
207
|
+
if (flags.effort) body.effort = flags.effort;
|
|
208
|
+
return body;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function _scanBody(flags) {
|
|
212
|
+
if (!flags.content) { console.error(' --content is required\n'); process.exit(1); }
|
|
213
|
+
return {
|
|
214
|
+
content: flags.content,
|
|
215
|
+
destination: flags.destination || 'public',
|
|
216
|
+
action: flags.do || 'send',
|
|
217
|
+
content_type: flags.type || 'message',
|
|
218
|
+
agent: _agentName(),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export async function runSend(flags) {
|
|
223
|
+
const apiKey = _requireApiKey();
|
|
224
|
+
_report(await api.evaluateEmail(_sendBody(flags), apiKey));
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export async function runSignin(flags) {
|
|
228
|
+
const apiKey = _requireApiKey();
|
|
229
|
+
_report(await api.evaluateLogin(_signinBody(flags), apiKey));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export async function runAction(flags) {
|
|
233
|
+
const apiKey = _requireApiKey();
|
|
234
|
+
_report(await api.evaluateAction(_actionBody(flags), apiKey));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export async function runModelCall(flags) {
|
|
238
|
+
const apiKey = _requireApiKey();
|
|
239
|
+
_report(await api.evaluateModel(_modelCallBody(flags), apiKey));
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export async function runScan(flags) {
|
|
243
|
+
const apiKey = _requireApiKey();
|
|
244
|
+
const result = await api.evaluateSecrets(_scanBody(flags), apiKey);
|
|
245
|
+
_report(result);
|
|
246
|
+
if (result.detection_count) {
|
|
247
|
+
console.log(` ${result.detection_count} thing(s) detected, highest severity: ${result.highest_severity || 'n/a'}`);
|
|
248
|
+
(result.detections || []).forEach(d => console.log(` - ${d.type} (${d.severity}): ${d.preview}`));
|
|
249
|
+
console.log();
|
|
250
|
+
}
|
|
251
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Every checkpoint the product has needs an MCP tool, or an MCP-connected
|
|
2
|
+
// agent - which is how most users connect - simply cannot reach it. Not "the
|
|
3
|
+
// agent chose not to call it": there is nothing to call.
|
|
4
|
+
//
|
|
5
|
+
// Found live, 2026-08-15: asked a connected agent to leak an AWS key and to
|
|
6
|
+
// bulk-delete prod rows, in an ordinary conversation, no mention of Troxy.
|
|
7
|
+
// The delete got refused (evaluate_action exists). The key-leak request was
|
|
8
|
+
// just carried out - not because the agent ignored a guardrail, but because
|
|
9
|
+
// evaluate_secrets had no MCP tool at all. Every other checkpoint
|
|
10
|
+
// (payment/email/login/action/model) had one; secrets did not. The backend's
|
|
11
|
+
// own other_checkpoints discovery (every /evaluate* response advertises the
|
|
12
|
+
// other five) did mention it, but as a raw HTTP URL - useless to an agent
|
|
13
|
+
// whose only way to reach Troxy is through the tools this file defines.
|
|
14
|
+
//
|
|
15
|
+
// model-checkpoint.test.js already covers this exact wiring for model/
|
|
16
|
+
// report_model_usage specifically, following a near-identical 2026-08-06
|
|
17
|
+
// incident (this is the second time a checkpoint has shipped on the API and
|
|
18
|
+
// dashboard without its MCP tool). This file generalizes the check across
|
|
19
|
+
// every checkpoint so a sixth one added later can't repeat it.
|
|
20
|
+
import { test } from 'node:test';
|
|
21
|
+
import assert from 'node:assert';
|
|
22
|
+
import { readFileSync } from 'node:fs';
|
|
23
|
+
import { fileURLToPath } from 'node:url';
|
|
24
|
+
import { dirname, join } from 'node:path';
|
|
25
|
+
|
|
26
|
+
const src = readFileSync(
|
|
27
|
+
join(dirname(fileURLToPath(import.meta.url)), '..', 'mcp-server.js'),
|
|
28
|
+
'utf8',
|
|
29
|
+
);
|
|
30
|
+
const apiSrc = readFileSync(
|
|
31
|
+
join(dirname(fileURLToPath(import.meta.url)), '..', 'api.js'),
|
|
32
|
+
'utf8',
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
// One MCP tool per checkpoint, and the /evaluate* path each one must reach.
|
|
36
|
+
// Deliberately a literal, not derived from mcp-server.js itself - a test
|
|
37
|
+
// that reads its expectation from the code under test cannot fail the way
|
|
38
|
+
// this one needs to, which is exactly how secrets went unnoticed despite
|
|
39
|
+
// model-checkpoint.test.js already existing as a precedent for this check.
|
|
40
|
+
const CHECKPOINTS = {
|
|
41
|
+
evaluate_payment: '/evaluate',
|
|
42
|
+
evaluate_email: '/evaluate/email',
|
|
43
|
+
evaluate_login: '/evaluate/login',
|
|
44
|
+
evaluate_action: '/evaluate/action',
|
|
45
|
+
evaluate_model: '/evaluate/model',
|
|
46
|
+
evaluate_secrets: '/evaluate/secrets',
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
test('every checkpoint has an MCP tool with a handler', () => {
|
|
50
|
+
for (const name of Object.keys(CHECKPOINTS)) {
|
|
51
|
+
assert.ok(
|
|
52
|
+
src.includes(`name: '${name}'`),
|
|
53
|
+
`${name} is not in the MCP tool list - an MCP-connected agent has no way to call it at all`,
|
|
54
|
+
);
|
|
55
|
+
assert.ok(
|
|
56
|
+
src.includes(`toolName === '${name}'`) || (name === 'evaluate_payment' && src.includes("toolName !== 'evaluate_payment'")),
|
|
57
|
+
`${name} is advertised but has no handler, so calling it does nothing`,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('every checkpoint tool is wired to its real endpoint in the api client', () => {
|
|
63
|
+
for (const [name, path] of Object.entries(CHECKPOINTS)) {
|
|
64
|
+
assert.ok(
|
|
65
|
+
apiSrc.includes(`'${path}'`),
|
|
66
|
+
`${name} has no api.js wrapper for ${path} - the MCP tool would exist with nothing behind it`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('evaluate_secrets is exported from mcp-server.js\'s own api bindings, not just referenced', () => {
|
|
72
|
+
// The other tools each get a one-line local binding
|
|
73
|
+
// (`const evaluateAction = (body, apiKey) => api.evaluateAction(...)`)
|
|
74
|
+
// before the tool list. secrets needs the same, or the handler below
|
|
75
|
+
// throws a ReferenceError the first time an agent actually calls it -
|
|
76
|
+
// which the two tests above cannot see, since they only check that the
|
|
77
|
+
// NAME 'evaluateSecrets' appears somewhere, not that it resolves.
|
|
78
|
+
assert.match(src, /const evaluateSecrets\s*=\s*\(body, apiKey\)\s*=>\s*api\.evaluateSecrets\(body, apiKey\)/);
|
|
79
|
+
});
|
|
@@ -242,6 +242,10 @@ test('no new tool was introduced for the chat path', () => {
|
|
|
242
242
|
const toolsSeg = src.slice(src.indexOf('tools: ['), src.indexOf('server.setRequestHandler(CallToolRequestSchema'));
|
|
243
243
|
const toolNames = [...toolsSeg.matchAll(/name: '([a-z_]+)'/g)].map(m => m[1]);
|
|
244
244
|
const unique = new Set(toolNames);
|
|
245
|
-
|
|
245
|
+
// 9, not this test's original 8: evaluate_secrets was added afterward
|
|
246
|
+
// (2026-08-15, mcp-checkpoints.test.js) to close a real gap, not to give
|
|
247
|
+
// chat its own tool - the assertion this test cares about is still that no
|
|
248
|
+
// *tenth*, chat-specific tool exists.
|
|
249
|
+
assert.strictEqual(unique.size, 9, 'a tool was added or removed for this change');
|
|
246
250
|
assert.ok(unique.has('evaluate_model') && unique.has('report_model_usage'));
|
|
247
251
|
});
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
// `policies create` used to know about exactly three fields - amount,
|
|
2
|
+
// merchant_name, tx_per_day - because payment was the whole product when it
|
|
3
|
+
// was written. These tests cover the domain-aware rewrite: every field in
|
|
4
|
+
// every checkpoint should be reachable, a field that belongs to more than one
|
|
5
|
+
// domain should require an explicit --domain rather than guessing, and the
|
|
6
|
+
// three condition-building paths (single/--and/--or, --json, --template)
|
|
7
|
+
// should each produce exactly the request body the server expects - checked
|
|
8
|
+
// directly against the built object, not by diffing console output.
|
|
9
|
+
import { test } from 'node:test';
|
|
10
|
+
import assert from 'node:assert';
|
|
11
|
+
import { readFileSync } from 'node:fs';
|
|
12
|
+
import { fileURLToPath } from 'node:url';
|
|
13
|
+
import { dirname, join } from 'node:path';
|
|
14
|
+
import { _buildConditionsFromFlags, _findTemplate } from '../policies.js';
|
|
15
|
+
|
|
16
|
+
// _buildConditionsFromFlags and _findTemplate call process.exit(1) on a bad
|
|
17
|
+
// input rather than throwing - this makes that path testable without ending
|
|
18
|
+
// the test process, by turning the exit into a catchable error. console.error
|
|
19
|
+
// still prints its message, matching how a user would actually see it.
|
|
20
|
+
function _expectExit(fn) {
|
|
21
|
+
const orig = process.exit;
|
|
22
|
+
process.exit = (code) => { throw new Error(`__EXIT_${code}__`); };
|
|
23
|
+
try {
|
|
24
|
+
fn();
|
|
25
|
+
throw new Error('expected process.exit to be called, but it was not');
|
|
26
|
+
} catch (e) {
|
|
27
|
+
if (!/^__EXIT_\d+__$/.test(e.message)) throw e;
|
|
28
|
+
} finally {
|
|
29
|
+
process.exit = orig;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
test('a payment field with no --domain still infers payment (backward compat)', () => {
|
|
34
|
+
const r = _buildConditionsFromFlags(
|
|
35
|
+
{ action: 'BLOCK', field: 'amount', operator: 'gt', value: '100' }, {}, { requireAction: true },
|
|
36
|
+
);
|
|
37
|
+
assert.strictEqual(r.domain, 'payment');
|
|
38
|
+
assert.deepStrictEqual(r.conditions, [{ field: 'amount', operator: 'gt', value: '100' }]);
|
|
39
|
+
assert.deepStrictEqual(r.or_conditions, []);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('a non-payment field works with an explicit --domain - this is the whole point of the rewrite', () => {
|
|
43
|
+
const r = _buildConditionsFromFlags(
|
|
44
|
+
{ action: 'ESCALATE', domain: 'access', field: 'login_action', operator: 'eq', value: 'signup' }, {}, { requireAction: true },
|
|
45
|
+
);
|
|
46
|
+
assert.strictEqual(r.domain, 'access');
|
|
47
|
+
assert.deepStrictEqual(r.conditions, [{ field: 'login_action', operator: 'eq', value: 'signup' }]);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('domain aliases resolve (login -> access, the persisted-column naming trap)', () => {
|
|
51
|
+
const r = _buildConditionsFromFlags(
|
|
52
|
+
{ action: 'ESCALATE', domain: 'login', field: 'login_action', operator: 'eq', value: 'signup' }, {}, { requireAction: true },
|
|
53
|
+
);
|
|
54
|
+
assert.strictEqual(r.domain, 'access');
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('a field shared by two domains (destination) requires an explicit --domain', () => {
|
|
58
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
59
|
+
{ action: 'BLOCK', field: 'destination', operator: 'eq', value: 'public' }, {}, { requireAction: true },
|
|
60
|
+
));
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('an operator the field does not support is rejected before it reaches the server', () => {
|
|
64
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
65
|
+
{ action: 'BLOCK', domain: 'payment', field: 'amount', operator: 'contains', value: '100' }, {}, { requireAction: true },
|
|
66
|
+
));
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test('a boolean field rejects anything but true/false', () => {
|
|
70
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
71
|
+
{ action: 'BLOCK', domain: 'secrets', field: 'has_api_key', operator: 'eq', value: 'yes' }, {}, { requireAction: true },
|
|
72
|
+
));
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('an enum field rejects a value the checkpoint never reports', () => {
|
|
76
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
77
|
+
{ action: 'BLOCK', domain: 'access', field: 'login_action', operator: 'eq', value: 'signin' }, {}, { requireAction: true },
|
|
78
|
+
));
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('repeated --and builds one AND block, no branches', () => {
|
|
82
|
+
const r = _buildConditionsFromFlags(
|
|
83
|
+
{ action: 'BLOCK', domain: 'destructive', field: 'action_verb', operator: 'eq', value: 'export' },
|
|
84
|
+
{ and: ['field=resource,operator=contains,value=customer'] },
|
|
85
|
+
{ requireAction: true },
|
|
86
|
+
);
|
|
87
|
+
assert.deepStrictEqual(r.conditions, [
|
|
88
|
+
{ field: 'action_verb', operator: 'eq', value: 'export' },
|
|
89
|
+
{ field: 'resource', operator: 'contains', value: 'customer' },
|
|
90
|
+
]);
|
|
91
|
+
assert.deepStrictEqual(r.or_conditions, []);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test('repeated --or builds one branch per alternative, sharing --action', () => {
|
|
95
|
+
const r = _buildConditionsFromFlags(
|
|
96
|
+
{ action: 'BLOCK', domain: 'comms' },
|
|
97
|
+
{ or: [
|
|
98
|
+
'field=recipient_domain,operator=contains,value=gmail.com',
|
|
99
|
+
'field=recipient_domain,operator=contains,value=yahoo.com',
|
|
100
|
+
] },
|
|
101
|
+
{ requireAction: true },
|
|
102
|
+
);
|
|
103
|
+
assert.strictEqual(r.conditions.length, 0);
|
|
104
|
+
assert.strictEqual(r.or_conditions.length, 2);
|
|
105
|
+
assert.deepStrictEqual(r.or_conditions[0], { action: 'BLOCK', conditions: [{ field: 'recipient_domain', operator: 'contains', value: 'gmail.com' }] });
|
|
106
|
+
assert.deepStrictEqual(r.or_conditions[1], { action: 'BLOCK', conditions: [{ field: 'recipient_domain', operator: 'contains', value: 'yahoo.com' }] });
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test('--and conditions apply to every --or branch, not just the first', () => {
|
|
110
|
+
const r = _buildConditionsFromFlags(
|
|
111
|
+
{ action: 'BLOCK', domain: 'destructive', field: 'action_verb', operator: 'eq', value: 'delete' },
|
|
112
|
+
{ or: ['field=resource,operator=contains,value=prod', 'field=resource,operator=contains,value=staging'] },
|
|
113
|
+
{ requireAction: true },
|
|
114
|
+
);
|
|
115
|
+
assert.strictEqual(r.or_conditions.length, 2);
|
|
116
|
+
for (const branch of r.or_conditions) {
|
|
117
|
+
assert.ok(branch.conditions.some(c => c.field === 'action_verb'), 'the AND condition must be present in every branch');
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('a malformed --and (missing operator=) is rejected with a clear error, not a silent partial condition', () => {
|
|
122
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
123
|
+
{ action: 'BLOCK', domain: 'payment' }, { and: ['field=amount,value=100'] }, { requireAction: true },
|
|
124
|
+
));
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('--json is a full escape hatch and is still domain-validated', () => {
|
|
128
|
+
const r = _buildConditionsFromFlags(
|
|
129
|
+
{ action: 'ESCALATE', domain: 'model', json: JSON.stringify({ conditions: [{ field: 'output_price', operator: 'gt', value: '10' }] }) },
|
|
130
|
+
{}, { requireAction: true },
|
|
131
|
+
);
|
|
132
|
+
assert.strictEqual(r.domain, 'model');
|
|
133
|
+
assert.deepStrictEqual(r.conditions, [{ field: 'output_price', operator: 'gt', value: '10' }]);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test('--json rejects a condition the field spec would reject', () => {
|
|
137
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
138
|
+
{ action: 'BLOCK', domain: 'model', json: JSON.stringify({ conditions: [{ field: 'output_price', operator: 'contains', value: '10' }] }) },
|
|
139
|
+
{}, { requireAction: true },
|
|
140
|
+
));
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test('--json cannot be combined with --field', () => {
|
|
144
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
145
|
+
{ action: 'BLOCK', domain: 'payment', field: 'amount', operator: 'gt', value: '1', json: '{}' }, {}, { requireAction: true },
|
|
146
|
+
));
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test('--template applies the template\'s own domain, action and conditions', () => {
|
|
150
|
+
const r = _buildConditionsFromFlags({ template: 'Block all destructive actions' }, {}, { requireAction: true });
|
|
151
|
+
assert.strictEqual(r.domain, 'destructive');
|
|
152
|
+
assert.strictEqual(r.action, 'BLOCK');
|
|
153
|
+
assert.deepStrictEqual(r.conditions, []);
|
|
154
|
+
assert.deepStrictEqual(r.or_conditions, []);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('--template --value overrides the threshold on a single-condition template', () => {
|
|
158
|
+
const r = _buildConditionsFromFlags({ template: 'Block models over $10/MTok output', value: '25' }, {}, { requireAction: true });
|
|
159
|
+
assert.strictEqual(r.conditions[0].value, '25');
|
|
160
|
+
assert.strictEqual(r.conditions[0].field, 'output_price'); // unchanged
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test('--template --value is refused on a template with more than one condition', () => {
|
|
164
|
+
_expectExit(() => _buildConditionsFromFlags({ template: 'Escalate making a file public', value: '5' }, {}, { requireAction: true }));
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test('--template --action overrides the template default', () => {
|
|
168
|
+
const r = _buildConditionsFromFlags({ template: 'Escalate making a file public', action: 'BLOCK' }, {}, { requireAction: true });
|
|
169
|
+
assert.strictEqual(r.action, 'BLOCK');
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test('--template cannot be combined with --field', () => {
|
|
173
|
+
_expectExit(() => _buildConditionsFromFlags(
|
|
174
|
+
{ template: 'Block all destructive actions', field: 'amount', operator: 'gt', value: '1' }, {}, { requireAction: true },
|
|
175
|
+
));
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test('an unknown template name exits with suggestions rather than a stack trace', () => {
|
|
179
|
+
_expectExit(() => _findTemplate('Block all destructve actions')); // typo
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
test('a template is found case-insensitively', () => {
|
|
183
|
+
const t = _findTemplate('block all destructive actions');
|
|
184
|
+
assert.strictEqual(t.label, 'Block all destructive actions');
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
// ── _displayAction: the raw `action` column is stale on essentially every
|
|
188
|
+
// dashboard-built policy; effective_action (added server-side, see
|
|
189
|
+
// troxy-tf-live's policies.py _effective_action) is the resolved value. This
|
|
190
|
+
// is only a fallback for a server that hasn't deployed the field yet.
|
|
191
|
+
|
|
192
|
+
test('_displayAction prefers effective_action when present', async () => {
|
|
193
|
+
const { _displayAction } = await import('../policies.js');
|
|
194
|
+
assert.strictEqual(_displayAction({ action: 'ESCALATE', effective_action: 'BLOCK' }), 'BLOCK');
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
test('_displayAction falls back to the raw action against an older server', async () => {
|
|
198
|
+
const { _displayAction } = await import('../policies.js');
|
|
199
|
+
assert.strictEqual(_displayAction({ action: 'BLOCK' }), 'BLOCK');
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
// ── --describe: handle_ai_policy only knows a binary payment/comms allowlist,
|
|
203
|
+
// gated on is_email. Source-inspection style, matching model-checkpoint.test.js -
|
|
204
|
+
// this branch needs a live login session + org AI key to drive end to end.
|
|
205
|
+
|
|
206
|
+
test('--describe sends is_email when --domain comms is requested', () => {
|
|
207
|
+
const src = readFileSync(join(dirname(fileURLToPath(import.meta.url)), '..', 'policies.js'), 'utf8');
|
|
208
|
+
const start = src.indexOf('if (flags.describe)');
|
|
209
|
+
const chunk = src.slice(start, start + 2000);
|
|
210
|
+
assert.ok(
|
|
211
|
+
/is_email:\s*describeDomain\s*===\s*'comms'/.test(chunk),
|
|
212
|
+
'--describe must forward is_email to api.aiPolicy, or every request silently uses the payment field allowlist regardless of what was described',
|
|
213
|
+
);
|
|
214
|
+
assert.ok(
|
|
215
|
+
/\['payment', 'comms'\]\.includes\(describeDomain\)/.test(chunk),
|
|
216
|
+
'--describe must reject any domain the AI builder does not actually support, rather than silently mis-drafting against the payment allowlist',
|
|
217
|
+
);
|
|
218
|
+
});
|