troxy-cli 1.13.0 → 1.14.0

package/README.md

@@ -67,6 +67,24 @@ npx troxy-cli <command>
 
 The CLI also ships an MCP server (`src/mcp-server.js`) that exposes Troxy as a tool for Claude and other MCP-compatible agents.
 
+Checkpoints an agent can call:
+
+| Tool | Call it before |
+|------|----------------|
+| `evaluate_payment` | paying, buying, transferring funds |
+| `confirm_payment` | (after) reporting whether the charge succeeded |
+| `evaluate_email` | sending an email or message |
+| `evaluate_action` | deleting, dropping, force-pushing, revoking |
+| `evaluate_login` | logging into or signing up for a site |
+| `evaluate_model` | running a task on a model, and on any model or effort switch |
+| `report_model_usage` | (after) reporting the real token total |
+| `wait_for_payment_approval` | (after ESCALATE) waiting on the owner's decision |
+
+The model checkpoint is the only one that is a required pair. `evaluate_model`
+records what a run is expected to cost; `report_model_usage` records what it
+actually cost. Without the second call Troxy holds only the agent's own
+prediction, and the dashboard marks that agent's figures as unverified.
+
 ## Auth flow
 
 `troxy login` uses a device-code flow:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "troxy-cli",
-  "version": "1.13.0",
+  "version": "1.14.0",
   "description": "Control layer for AI agents: check payments, emails, logins and destructive actions against your policies",
   "homepage": "https://troxy.io",
   "bugs": {

package/src/api.js

@@ -74,6 +74,11 @@ export const api = {
   evaluateEmail: (body, apiKey) => request('POST', '/evaluate/email', { apiKey, body }),
   evaluateLogin: (body, apiKey) => request('POST', '/evaluate/login', { apiKey, body }),
   evaluateAction: (body, apiKey) => request('POST', '/evaluate/action', { apiKey, body }),
+  // The model checkpoint is two calls: ask before the run, report after it.
+  // Only the pair produces usable data. The estimate alone is the agent's own
+  // guess, and Token Optimization says so on every agent that never reports.
+  evaluateModel: (body, apiKey) => request('POST', '/evaluate/model', { apiKey, body }),
+  reportModelUsage: (body, apiKey) => request('POST', '/evaluate/model/complete', { apiKey, body }),
   selfRevoke: (apiKey) => request('POST', '/mcp/self-revoke', { apiKey }),
   revokeAllOthers: (apiKey) => request('POST', '/mcp/revoke-all-others', { apiKey }),
   confirmPayment: (auditId, body, apiKey) => request('POST', `/payments/${auditId}/confirm`, { apiKey, body }),

package/src/mcp-server.js

@@ -10,6 +10,8 @@ const confirmPayment = (auditId, body, apiKey) => api.confirmPayment(auditId, bo
 const evaluateEmail   = (body, apiKey) => api.evaluateEmail(body, apiKey);
 const evaluateLogin   = (body, apiKey) => api.evaluateLogin(body, apiKey);
 const evaluateAction  = (body, apiKey) => api.evaluateAction(body, apiKey);
+const evaluateModel   = (body, apiKey) => api.evaluateModel(body, apiKey);
+const reportModelUsage = (body, apiKey) => api.reportModelUsage(body, apiKey);
 
 export async function runMcp() {
   const config    = loadConfig();
@@ -44,7 +46,12 @@ export async function runMcp() {
         'After every ALLOW or NOTIFY decision, attempt the payment, then call confirm_payment with the audit_id and the outcome (success/failed/cancelled). Always include a reason when the charge fails. ' +
         'You MUST also call the evaluate_email tool before sending any email on the user\'s behalf. Always include the full plaintext "body" argument, exactly as you intend to send it — Troxy policies inspect the body for secrets and confidential content, and cannot do so if it is omitted, summarized, or truncated. When the email goes to more than one person (a reply-all, a group, a bulk send), also pass every To/Cc/Bcc address in "recipients" so mass-email policies can apply. Never send an email without first receiving an ALLOW or NOTIFY decision. If BLOCK, do not send it. If ESCALATE, call wait_for_payment_approval with the returned approval_token (same wait mechanism payments use), then call evaluate_email again with the approval_token once approved. ' +
         'You MUST also call the evaluate_action tool before any destructive or hard-to-undo action on the user\'s behalf: deleting files or records, dropping or truncating a table, force-pushing or deleting a branch, destroying infrastructure, revoking access, or overwriting data. Give the verb, what it affects, how many items, and whether it can be undone. Never carry the action out without first receiving an ALLOW or NOTIFY decision. ' +
-        'You MUST also call the evaluate_login tool before logging into or signing up for any website on the user\'s behalf, giving the site\'s domain and whether it is a login or a signup. Never submit login credentials or a signup form without first receiving an ALLOW or NOTIFY decision. If BLOCK, do not proceed. If ESCALATE, call wait_for_payment_approval with the returned approval_token, then call evaluate_login again with the approval_token once approved.',
+        'You MUST also call the evaluate_login tool before logging into or signing up for any website on the user\'s behalf, giving the site\'s domain and whether it is a login or a signup. Never submit login credentials or a signup form without first receiving an ALLOW or NOTIFY decision. If BLOCK, do not proceed. If ESCALATE, call wait_for_payment_approval with the returned approval_token, then call evaluate_login again with the approval_token once approved. ' +
+        // The model checkpoint is the only one that is a pair. The estimate on
+        // its own is the agent's own prediction, and the dashboard labels every
+        // agent that never reports back as unverified, so the instruction has
+        // to make the second call as non-optional as the first.
+        'You MUST also call the evaluate_model tool when you begin a substantial task and whenever you switch model or effort level partway through it, giving the exact model id and how many tokens you expect to use. Do not run the task without first receiving an ALLOW or NOTIFY decision; if BLOCK, pick a cheaper or smaller model and call evaluate_model again with the new id. Then, when the task finishes, you MUST call report_model_usage with the audit_id from that decision and the real token total from your model API. Both halves are required: without the second call Troxy has only your prediction, and the account owner is shown that this agent\'s costs are unverified. Report the true number even when it is far above your estimate.',
     },
     { capabilities: { tools: {} } },
   );
@@ -255,6 +262,76 @@ export async function runMcp() {
           },
         },
       },
+      {
+        name: 'evaluate_model',
+        description:
+          'Evaluate whether running a task on a given AI model should be allowed, blocked, or escalated ' +
+          'based on your Troxy policies, and record what it is expected to cost. ' +
+          'Call this when you start a task and whenever you switch model or effort level mid-task. ' +
+          'Returns an audit_id. Keep it: you must pass it to report_model_usage when the task finishes.',
+        inputSchema: {
+          type: 'object',
+          required: ['model'],
+          properties: {
+            model: {
+              type: 'string',
+              description: 'The full model id you are about to use, e.g. "claude-sonnet-4.5", "gpt-4o-mini". Send the exact id, not a friendly name: Troxy prices per model, and generations of the same family differ (Claude 3 Opus costs three times Opus 4.x).',
+            },
+            estimated_tokens: {
+              type: 'number',
+              description: 'How many tokens you expect the task to use in total, input plus output. Troxy prices this itself, so an estimate in tokens is enough; do not convert it to dollars.',
+            },
+            effort: {
+              type: 'string',
+              enum: ['low', 'medium', 'high'],
+              description: 'How much work this task warrants. Report it honestly: "high effort on a simple task" is one of the things Troxy flags as waste.',
+            },
+            task: {
+              type: 'string',
+              description: 'One short line on what the model is being asked to do (optional). Shown to the account owner in the audit log.',
+            },
+            provider: {
+              type: 'string',
+              description: 'Model provider, e.g. "anthropic", "openai" (optional; inferred from the model id when omitted).',
+            },
+            agent: {
+              type: 'string',
+              description: 'Name of the agent running the task (optional).',
+            },
+            approval_token: {
+              type: 'string',
+              description: 'Approval token from a previous ESCALATE response. Include this to proceed after the user has approved.',
+            },
+          },
+        },
+      },
+      {
+        name: 'report_model_usage',
+        description:
+          'Report how many tokens a task ACTUALLY used, once it has finished. ' +
+          'Call this after every ALLOW or NOTIFY from evaluate_model, using the audit_id it returned. ' +
+          'Without it Troxy only has your prediction, and the account owner is told, per agent, that its figures are unverified. ' +
+          'Report the real number even when it is far above what you estimated: the gap is the point.',
+        inputSchema: {
+          type: 'object',
+          required: ['audit_id', 'actual_tokens'],
+          properties: {
+            audit_id: {
+              type: 'string',
+              description: 'The audit_id returned by evaluate_model for this task',
+            },
+            actual_tokens: {
+              type: 'number',
+              description: 'Total tokens the task really consumed, input plus output. Use the usage figures your model API returned, not an estimate.',
+            },
+            effort: {
+              type: 'string',
+              enum: ['low', 'medium', 'high'],
+              description: 'The effort level the task actually took, if it differed from what you declared up front (optional).',
+            },
+          },
+        },
+      },
     ],
   }));
 
@@ -395,6 +472,60 @@ export async function runMcp() {
       return { content: [{ type: 'text', text: actionText }], isError: decision === 'BLOCK' };
     }
 
+    if (toolName === 'evaluate_model') {
+      if (agentName && !args.agent) args.agent = agentName;
+      let result;
+      try {
+        result = await evaluateModel(args, apiKey);
+      } catch (err) {
+        return { content: [{ type: 'text', text: `Troxy error: ${err.message}` }], isError: true };
+      }
+      if (result.error) {
+        return { content: [{ type: 'text', text: `Troxy error: ${result.error}` }], isError: true };
+      }
+      const { decision, reason, audit_id, approval_token } = result;
+      const what = `${args.model}${args.effort ? ` at ${args.effort} effort` : ''}`;
+      // Every non-blocked branch repeats the audit_id and the instruction to
+      // report back. Half the value of this checkpoint is the second call, and
+      // an agent that is only told "approved" has no reason to make it.
+      const followUp = ` When the task finishes, call report_model_usage(audit_id="${audit_id}", actual_tokens=<real total>).`;
+      let modelText;
+      switch (decision) {
+        case 'ALLOW':
+        case 'NOTIFY':
+          modelText = `✓ Approved: ${what}.${reason ? ` ${reason}` : ''} You may run it.${followUp} (audit: ${audit_id})`;
+          break;
+        case 'BLOCK':
+          modelText = `✗ Blocked: ${what}.${reason ? ` ${reason}` : ''} Do not use this model for this task. Choose a cheaper or smaller model and call evaluate_model again with the new model id. (audit: ${audit_id})`;
+          break;
+        case 'ESCALATE':
+          modelText = `⏳ ${what} requires human approval; a request has been sent to the account owner.\n\nApproval token: ${approval_token}\n\nNow call wait_for_payment_approval(approval_token="${approval_token}") to automatically detect approval, then call evaluate_model again with the same arguments PLUS this approval_token. Do not run the task until it returns approved.`;
+          break;
+        default:
+          modelText = JSON.stringify(result);
+      }
+      return { content: [{ type: 'text', text: modelText }], isError: decision === 'BLOCK' };
+    }
+
+    if (toolName === 'report_model_usage') {
+      let result;
+      try {
+        result = await reportModelUsage(args, apiKey);
+      } catch (err) {
+        return { content: [{ type: 'text', text: `Troxy error: ${err.message}` }], isError: true };
+      }
+      if (result.error) {
+        return { content: [{ type: 'text', text: `Troxy error: ${result.error}` }], isError: true };
+      }
+      const cost = typeof result.actual_cost === 'number'
+        ? ` Cost: $${result.actual_cost.toFixed(4)}.`
+        : '';
+      return {
+        content: [{ type: 'text', text: `✓ Usage recorded: ${result.actual_tokens} tokens.${cost} (audit: ${args.audit_id})` }],
+        isError: false,
+      };
+    }
+
     if (toolName === 'evaluate_login') {
       if (agentName && !args.agent) args.agent = agentName;
       let result;

package/src/tests/model-checkpoint.test.js

@@ -0,0 +1,100 @@
+// The model checkpoint has to be reachable, and it has to be a pair.
+//
+// Until 2026-08-06 the API had /evaluate/model and /evaluate/model/complete and
+// the MCP server exposed neither. Any agent connected over MCP, which is how
+// most users connect, could not report model usage at all, so Token
+// Optimization was permanently empty for them and nothing on the page said why.
+// Verified on the live database that day: every model row on a real account was
+// hand-made test data. Nothing had ever reported on its own.
+//
+// The subtler half is the report. An estimate on its own is the agent's own
+// prediction of what a run would cost, which is exactly the number the
+// dashboard refuses to present as fact. A tool list that offers the check
+// without the report reintroduces that gap while looking complete.
+
+import { test } from 'node:test';
+import assert from 'node:assert';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const src = readFileSync(
+  join(dirname(fileURLToPath(import.meta.url)), '..', 'mcp-server.js'),
+  'utf8',
+);
+const apiSrc = readFileSync(
+  join(dirname(fileURLToPath(import.meta.url)), '..', 'api.js'),
+  'utf8',
+);
+
+test('both halves of the model checkpoint are exposed as tools', () => {
+  for (const name of ['evaluate_model', 'report_model_usage']) {
+    assert.ok(
+      src.includes(`name: '${name}'`),
+      `${name} is not in the tool list, so an MCP agent cannot call it and ` +
+      `Token Optimization stays empty for every agent connected this way`,
+    );
+    assert.ok(
+      src.includes(`toolName === '${name}'`),
+      `${name} is advertised but has no handler, so calling it does nothing`,
+    );
+  }
+});
+
+test('both endpoints are wired in the api client', () => {
+  assert.ok(apiSrc.includes("'/evaluate/model'"), 'evaluate/model not wired');
+  assert.ok(
+    apiSrc.includes("'/evaluate/model/complete'"),
+    'evaluate/model/complete not wired, so actuals can never be reported',
+  );
+});
+
+test('the server instructions tell the agent to do both', () => {
+  // Checkpoints are cooperative: an agent only calls in because it was told
+  // to. A tool nobody is instructed to use is a tool nobody uses.
+  const instructions = src.slice(0, src.indexOf('capabilities'));
+  assert.ok(
+    instructions.includes('evaluate_model'),
+    'nothing in the server instructions tells the agent to check its model, ' +
+    'so the tool exists and never gets called',
+  );
+  assert.ok(
+    instructions.includes('report_model_usage'),
+    'the instructions ask for the estimate but never for the actual, which ' +
+    'is the half that makes the number trustworthy',
+  );
+});
+
+test('an approved decision repeats the audit_id and asks for the report', () => {
+  // The agent has to carry the audit_id from one call to the other. If the
+  // ALLOW response does not hand it back along with the instruction, the
+  // second call depends on the agent having remembered a tool description.
+  const handler = src.slice(src.indexOf("toolName === 'evaluate_model'"));
+  const allowBranch = handler.slice(0, handler.indexOf("toolName === 'report_model_usage'"));
+  assert.ok(
+    allowBranch.includes('report_model_usage(audit_id='),
+    'the ALLOW/NOTIFY response does not tell the agent how to report back ' +
+    'with the audit_id it was just given',
+  );
+});
+
+test('a blocked model tells the agent what to do instead', () => {
+  const handler = src.slice(src.indexOf("toolName === 'evaluate_model'"));
+  const blocked = handler.slice(handler.indexOf("case 'BLOCK'"));
+  assert.ok(
+    /cheaper or smaller model/.test(blocked.slice(0, 400)),
+    'BLOCK says no without saying what to do next, so the agent either ' +
+    'stops or retries the same model',
+  );
+});
+
+test('the model id is asked for exactly, not as a friendly name', () => {
+  // Generations of one family do not share a price: Claude 3 Opus is $15/$75
+  // against $5/$25 for Opus 4.x. "claude opus" prices as the wrong one.
+  const schema = src.slice(src.indexOf("name: 'evaluate_model'"));
+  assert.ok(
+    /exact id, not a friendly name/.test(schema.slice(0, 2000)),
+    'the model description does not insist on the exact id, so agents will ' +
+    'send family names and be priced as the wrong generation',
+  );
+});