rightmodeler 0.3.0 → 0.4.0

package/dist-bundle/proxy/proxy-runtime.mjs

@@ -40,6 +40,12 @@ function isObject(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+// The same 1e-12 snap as roundUsd in ../budget.ts, which this runtime ships without:
+// lease sums then do not depend on the order requests settle in.
+function roundUsd(value) {
+  return Number(value.toFixed(12));
+}
+
 function parseConfig() {
   const rawSwapPolicy = jsonEnv("RM_SWAP_POLICY");
   if (
@@ -227,11 +233,11 @@ function loadState(spoolPath, checkpointPath) {
     state.lastAttemptGroup = Math.max(state.lastAttemptGroup, row.attemptGroup);
     state.groups.set(row.logicalCallId, row.attemptGroup);
     state.attemptIds.add(row.attemptId);
-    state.spentUsd += row.costUsd;
+    state.spentUsd = roundUsd(state.spentUsd + row.costUsd);
   }
 
   for (const reservation of state.reservations.values()) {
-    state.spentUsd += reservation.reservedUsd;
+    state.spentUsd = roundUsd(state.spentUsd + reservation.reservedUsd);
   }
 
   return state;
@@ -736,9 +742,10 @@ async function main() {
     const estimatedInputTokens = forwardedBody.length;
     const estimatedWorstCaseUsd =
       estimatedInputTokens * pricing.input + maxTokens * pricing.output;
-    const requiredLeaseUsd =
-      state.spentUsd + reservedUsd + estimatedWorstCaseUsd;
-    if (requiredLeaseUsd > config.lease.maxUsd) {
+    const requiredLeaseUsd = roundUsd(
+      state.spentUsd + reservedUsd + estimatedWorstCaseUsd,
+    );
+    if (requiredLeaseUsd > roundUsd(config.lease.maxUsd)) {
       appendRow(spoolPath, {
         kind: "blocked",
         runId: config.runId,
@@ -763,7 +770,7 @@ async function main() {
       return;
     }
 
-    reservedUsd += estimatedWorstCaseUsd;
+    reservedUsd = roundUsd(reservedUsd + estimatedWorstCaseUsd);
     const attemptId = nextAttemptId(state);
     const responseSpoolPath = join(streamDirectory, `${attemptId}.txt`);
     appendRow(spoolPath, {
@@ -892,10 +899,10 @@ async function main() {
         endedAt: new Date().toISOString(),
       });
       state.reservations.delete(attemptId);
-      state.spentUsd += leaseChargeUsd;
+      state.spentUsd = roundUsd(state.spentUsd + leaseChargeUsd);
       if (!outgoing.destroyed && !outgoing.writableEnded) outgoing.end();
     } finally {
-      reservedUsd -= estimatedWorstCaseUsd;
+      reservedUsd = roundUsd(reservedUsd - estimatedWorstCaseUsd);
     }
   }
 

package/docs/commands.md

@@ -61,6 +61,16 @@ Options:
   --base-url <url>                    OpenAI-compatible provider base URL
   --api-key-env <name>                environment variable containing the
                                       provider API key
+  --route <kind>                      where candidate replays run: api (the
+                                      --base-url endpoint), claude-login or
+                                      codex-login (the claude or codex CLI
+                                      signed in on this machine); a plan route
+                                      needs --judge-route (choices: "api",
+                                      "claude-login", "codex-login")
+  --judge-route <kind>                where the built-in judge runs: api,
+                                      claude-login or codex-login; use a vendor
+                                      other than the candidates' (choices:
+                                      "api", "claude-login", "codex-login")
   --max-cost-usd <amount>             optional hard spend cap in USD; omit to
                                       run uncapped so every case and judge cell
                                       completes
@@ -99,8 +109,8 @@ Options:
                                       "aggregate", "confirm", "report")
   --code-graph <path>                 Graphify graph.json for static code
                                       context in the report; never evidence
-  --yes                               accept the newest discovered trace without
-                                      prompting
+  --yes                               accept the newest discovered trace and the
+                                      saved model route without prompting
   -h, --help                          display help for command
 ```
 
@@ -121,6 +131,16 @@ Options:
   --base-url <url>                    OpenAI-compatible provider base URL
   --api-key-env <name>                environment variable containing the
                                       provider API key
+  --route <kind>                      where candidate replays run: api (the
+                                      --base-url endpoint), claude-login or
+                                      codex-login (the claude or codex CLI
+                                      signed in on this machine); a plan route
+                                      needs --judge-route (choices: "api",
+                                      "claude-login", "codex-login")
+  --judge-route <kind>                where the built-in judge runs: api,
+                                      claude-login or codex-login; use a vendor
+                                      other than the candidates' (choices:
+                                      "api", "claude-login", "codex-login")
   --max-cost-usd <amount>             optional hard spend cap in USD; omit to
                                       run uncapped so every case and judge cell
                                       completes
@@ -152,8 +172,8 @@ Options:
   --evaluator-gate-metric <name>      scorer metric used for release gates
   --evaluator-gate-threshold <value>  fallback pass threshold when the evaluator
                                       omits a pass decision
-  --yes                               accept the newest discovered trace without
-                                      prompting
+  --yes                               accept the newest discovered trace and the
+                                      saved model route without prompting
   --approved-run <digest>             scope projection to one merged approved
                                       swap
   -h, --help                          display help for command
@@ -251,6 +271,16 @@ Options:
   --base-url <url>                    OpenAI-compatible provider base URL
   --api-key-env <name>                environment variable containing the
                                       provider API key
+  --route <kind>                      where candidate replays run: api (the
+                                      --base-url endpoint), claude-login or
+                                      codex-login (the claude or codex CLI
+                                      signed in on this machine); a plan route
+                                      needs --judge-route (choices: "api",
+                                      "claude-login", "codex-login")
+  --judge-route <kind>                where the built-in judge runs: api,
+                                      claude-login or codex-login; use a vendor
+                                      other than the candidates' (choices:
+                                      "api", "claude-login", "codex-login")
   --max-cost-usd <amount>             optional hard spend cap in USD; omit to
                                       run uncapped so every case and judge cell
                                       completes
@@ -320,6 +350,16 @@ Options:
   --base-url <url>                    OpenAI-compatible provider base URL
   --api-key-env <name>                environment variable containing the
                                       provider API key
+  --route <kind>                      where candidate replays run: api (the
+                                      --base-url endpoint), claude-login or
+                                      codex-login (the claude or codex CLI
+                                      signed in on this machine); a plan route
+                                      needs --judge-route (choices: "api",
+                                      "claude-login", "codex-login")
+  --judge-route <kind>                where the built-in judge runs: api,
+                                      claude-login or codex-login; use a vendor
+                                      other than the candidates' (choices:
+                                      "api", "claude-login", "codex-login")
   --max-cost-usd <amount>             optional hard spend cap in USD; omit to
                                       run uncapped so every case and judge cell
                                       completes
@@ -585,7 +625,7 @@ print documentation packaged with this CLI
 Arguments:
   name        packaged document name (choices: "commands", "evaluators",
               "exit-codes", "gateways", "getting-started", "github",
-              "github-actions", "modeb")
+              "github-actions", "modeb", "model-routes")
 
 Options:
   -h, --help  display help for command

package/docs/evaluators.md

@@ -1,6 +1,6 @@
 # Evaluators
 
-The default evaluator is the built-in judge selected from the configured provider catalog. Candidate and reference families are excluded when choosing the judge. The exclusion applies per replayed call site: when the call sites of one traced family use models of different vendors (for example `openai/...` and `anthropic/...`), each call site's cases are judged by a model from neither the candidate's model family nor that call site's. Confirmation judges the recorded final output, so its judge also comes from outside the final step's model family.
+The default evaluator is the built-in judge selected from the configured provider catalog. Candidate and reference families are excluded when choosing the judge. The exclusion applies per replayed call site: when the call sites of one traced family use models of different vendors (for example `openai/...` and `anthropic/...`), each call site's cases are judged by a model from neither the candidate's model family nor that call site's. Confirmation judges the recorded final output, so its judge also comes from outside the final step's model family. In replay, when the recorded answer is one assistant message of text, the judge compares the candidate's answer with that text; any other recorded answer, such as one that calls tools, has parts other than text or holds several messages, is shown to the judge as recorded.
 
 An external evaluator is requested with `--evaluator <provider>` on `init`, `estimate`, `replay`, and `confirm`, where the provider is `braintrust`, `langfuse`, `langsmith`, or `promptfoo`. Every other `--evaluator-*` option is a usage error without `--evaluator`.
 

package/docs/exit-codes.md

@@ -6,7 +6,7 @@ Rightmodeler reserves exit codes `0` through `3` for machine-readable outcomes.
 
 - `0`: the command completed and no recommendation is being reported. Planning and partial `--through` runs also return `0` when successful.
 - `1`: a complete `init` or `report` found an actionable recommendation.
-- `2`: the run needs input at a resumable boundary, such as missing traces, a cancelled trace prompt, provider configuration, or required confirmation configuration.
+- `2`: the run needs input at a resumable boundary, such as missing traces, a cancelled trace prompt, provider configuration, required confirmation configuration, or a plan's usage limit (rerun after it resets).
 - `3`: the cost budget was reached at a resumable boundary.
 - `10` or greater: command-line or runtime failure.
 
@@ -54,13 +54,18 @@ Use `--output json` for one result object or `--output jsonl` for stage events f
 - `invalid_option` (exit `2`): correct the option and rerun; use `rightmodeler <command> --help` for accepted values.
 - `invalid_policy_file` (exit `2`): fix the named field in the `--policy` file and rerun; `qualityFloor` must be greater than 0.8 and less than 1, `shortlistTop` a positive integer, `allowModels` and `denyModels` arrays of model ids.
 - `invalid_pricing_file` (exit `2`): fix `--pricing-file` to map each model id to non-negative `input` and `output` USD per token and, optionally, a positive integer `maxOutputTokens`, then rerun.
+- `judge_family_unknown` (exit `2`): the catalog's model ids name no vendor, so the built-in judge could share a vendor with the candidate or the recorded model; use a gateway whose ids carry their vendor (`vendor/model`), or grade with `--evaluator`.
 - `missing_provider_configuration` (exit `2`): pass `--base-url <url>` and, if needed, `--api-key-env <environment-variable-name>` naming a populated variable.
 - `missing_traces_path` (exit `2`): pass `--traces <path>` pointing to an existing trace file or directory.
 - `mixed_trace_formats` (exit `2`): split the directory so every file uses the same trace format, or pass one file with `--traces`.
 - `modeb_cloud_unavailable` (exit `2`): install the optional sandbox SDK and set its credentials, or set `"backend": "docker"` in the `--modeb-config` file, then rerun.
+- `no_neutral_judge` (exit `2`): the catalog has no priced model from a vendor other than both the candidate's and the recorded model's, which the built-in judge needs; list or price one (a multi-vendor gateway, `--catalog-reference` or `--pricing-file`), run the judge through another vendor's CLI with `--judge-route`, or grade with `--evaluator`.
 - `no_priced_candidates` (exit `2`): point `--base-url` at a catalog that publishes per-token pricing, pass `--catalog-reference <url>`, expose priced LiteLLM `GET /model/info`, or pass `--pricing-file <path>`, then rerun.
 - `no_replayable_call_sites` (exit `2`): point `--repo` at a service with plain text completions, or add a matcher for a text call site, then rerun.
 - `not_git_repository` (exit `2`): run the command again from a Git repository with at least one commit.
+- `plan_cli_unavailable` (exit `2`): the CLI a plan route runs is missing, older than the verified version, or changed an output shape, or `CI` is set; install or update it (`claude update`, or `npm install -g @openai/codex@latest` for `codex`), unset `CI` on your own machine, or use an API route with `--base-url`.
+- `plan_login_required` (exit `2`): sign the CLI in to your plan (`claude auth login`, or `codex login` with ChatGPT) and remove any API key setting it would use, or use an API route with `--base-url`; finished calls are kept.
+- `plan_usage_limit` (exit `2`): a plan you are signed in to reached its usage limit; rerun the same command after the reset time in the message, and completed replay and judge calls are kept and not repeated; or choose a route that does not use this plan with `--route` or `--judge-route`.
 - `stage_not_completed` (exit `2`): run `rightmodeler init --through <stage>` first, then rerun the command.
 - `unusable_trace_input` (exit `2`): the selected discovered trace could not be adapted; rerun and choose a different trace file.
 - `usage_error` (exit `10`): the command line is invalid; `message` carries the parser text.

package/docs/gateways.md

@@ -69,7 +69,7 @@ On Kubernetes (Kubernetes 1.32 or newer, Envoy Gateway 1.8.1 or newer, Helm char
 
 ## Bifrost
 
-Verified on the open-source Bifrost gateway transports/v2.2.1 (Apache 2.0, `maximhq/bifrost:v2.2.1`); pin the image, because releases arrive weekly. Enterprise features are a separate image and are not covered here.
+Verified on the open-source Bifrost gateway transports/v2.2.1 (Apache 2.0, `maximhq/bifrost:v2.2.1`); pin the image, because releases arrive weekly. Enterprise features ship in a separate image; this guide uses the open-source one.
 
 As a replay route:
 

package/docs/getting-started.md

@@ -7,7 +7,13 @@ Rightmodeler analyzes recorded model calls, replays them against cheaper candida
 - Node.js 24 or newer.
 - A Git repository to analyze.
 - Trace input in a supported format.
-- An OpenAI-compatible provider base URL and the name of an environment variable containing its API key before replay begins. Its `/v1/models` catalog should publish per-token pricing. OpenRouter and Vercel AI Gateway do. For a LiteLLM endpoint, Rightmodeler can fall back to `GET /model/info`; for a gateway that lists bare model ids, pass `--catalog-reference`; for bare OpenAI or another unpriced endpoint, pass `--pricing-file`.
+- An OpenAI-compatible provider base URL and the name of an environment variable containing its API key before replay begins. Its `/v1/models` catalog should publish per-token pricing. OpenRouter and Vercel AI Gateway do. For a LiteLLM endpoint, Rightmodeler can fall back to `GET /model/info`; for a gateway that lists bare model ids or a direct OpenAI or Anthropic key, pass `--catalog-reference`; for another unpriced endpoint, pass `--pricing-file`.
+
+Instead of an API key, candidates or the judge can run through the `claude`
+CLI you are signed in to on this machine, under your Claude plan:
+`--route claude-login` with a `--judge-route` from another vendor. Run
+`rightmodeler docs model-routes` for what a plan route measures, what it costs
+your plan, and its safeguards.
 
 Supported trace sources are OTel GenAI, AI SDK telemetry, OpenAI JSONL,
 Langfuse, Braintrust, LangSmith, OpenInference, Helicone, Bifrost, W&B Weave,
@@ -21,6 +27,13 @@ Run this from the repository you want to analyze:
 npx rightmodeler init
 ```
 
+Before it looks for traces, an interactive `init` or `estimate` that will reach
+replay asks how to call models: through the `claude` or `codex` CLI signed in on
+this machine, or through an API key for OpenRouter, Vercel AI Gateway, OpenAI,
+Anthropic or another OpenAI-compatible endpoint. It prints the equivalent flags
+and saves the answer as the next default, which `--yes` applies without asking.
+`npx rightmodeler docs model-routes` describes the choice.
+
 Rightmodeler checks conventional local trace files, Claude Code transcripts for
 the repository, and Codex sessions whose recorded working directory matches the
 repository. In an interactive terminal it lists matches newest-first with an
@@ -121,9 +134,8 @@ Graph edges are never replay trials, runtime proof, or quality evidence, and the
 
 Rightmodeler reads per-token pricing from the model catalog. When every catalog
 entry has null pricing and no `--pricing-file` is set, it requests LiteLLM
-`GET /model/info` on the same host. Use `--pricing-file` for bare OpenAI
-endpoints or when `/model/info` has no usable per-token costs; file entries
-override provider pricing.
+`GET /model/info` on the same host. Use `--pricing-file` when `/model/info` has
+no usable per-token costs; file entries override provider pricing.
 
 Some gateways answer `/v1/models` with bare model ids: Envoy AI Gateway lists
 the models a route declares, and Bifrost lists custom providers with ids and
@@ -169,6 +181,33 @@ Without usable pricing from the catalog, a catalog reference, LiteLLM
 instead of reporting zero cost. The judge must be priced too, so price at least
 one model from a family other than the current model's and the candidate's.
 
+### Direct OpenAI and Anthropic keys
+
+Point `--base-url` at `https://api.openai.com/v1` or
+`https://api.anthropic.com/v1` and name the variable that holds the key with
+`--api-key-env`. Neither vendor's model list publishes prices, so also pass
+`--catalog-reference https://ai-gateway.vercel.sh/v1/models`. Rightmodeler takes
+the vendor from these two hosts: their bare ids are that vendor's models and
+join the reference as `openai/<id>` or `anthropic/<id>`. Dated and dotted forms
+of a name match within one vendor, in the reference and in your traces, so
+`claude-haiku-4-5-20251001` matches `anthropic/claude-haiku-4.5`. For Anthropic,
+rightmodeler sends `anthropic-version: 2023-06-01` (a
+`--header 'anthropic-version: ...'` value wins), reads every page of the model
+list, and never asks for structured output, which Anthropic's OpenAI-compatible
+endpoint ignores. For OpenAI, rightmodeler caps a reply's length with
+`max_completion_tokens`, which OpenAI's chat reference names in place of the
+deprecated `max_tokens`; every other host still gets `max_tokens`.
+
+The built-in judge must come from a vendor other than both the candidate's and
+the recorded model's, and rightmodeler checks this before any paid call, also
+when an unreachable `--evaluator` falls back to the built-in judge. One vendor's
+key lists only that vendor's models, so a run on it alone stops with
+`no_neutral_judge` unless the judge runs through the other vendor's CLI you are
+signed in to (`--judge-route`, see `rightmodeler docs model-routes`) or your own
+evaluator grades the replays. Bare ids from any other host name no vendor, and
+a run on them stops with `judge_family_unknown`; use a gateway whose ids carry
+their vendor (`vendor/model`), or `--evaluator`.
+
 ## Which model answered
 
 Rightmodeler checks every replay and judge response, and in Mode B every response to a step whose model it sets, before it counts. It records the model the response names, and it leaves a response out of the evidence when that model is not the one it asked for, when a gateway reports that it answered from its cache (Portkey's `x-portkey-cache-status: HIT`, Bifrost's `cache_debug.cache_hit`), or when a gateway reports that it changed the request (a Portkey hook with `transformed: true`, Bifrost's compat plugin dropping parameters). Such a response is recorded with `attribution: "substituted"`, is never graded, and counts as `attribution_substituted`; more than 5% of a family's replays substituted abstains the family. A `replay_responses_substituted` warning counts them.

package/docs/github-actions.md

@@ -48,7 +48,7 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  RIGHTMODELER_VERSION: "0.3.0"
+  RIGHTMODELER_VERSION: "0.4.0"
   RIGHTMODELER_TRACES: traces
   RIGHTMODELER_MAX_COST_USD: "5"
   RM_ANNOTATE: |
@@ -305,7 +305,7 @@ jobs:
 
 ## Optional: a GitHub App token
 
-Not tested live: rightmodeler's acceptance runs of this workflow use `GITHUB_TOKEN` only. A GitHub App installation token lets the draft's `pull_request` workflows start without approval, and the draft is authored by `<app-slug>[bot]`. To use one:
+The workflow above runs on the built-in `GITHUB_TOKEN`, the supported default. A GitHub App installation token is an optional upgrade: the draft's `pull_request` workflows start without approval, and the draft is authored by `<app-slug>[bot]`. To use one:
 
 1. Create a GitHub App with the permissions in [GitHub](github.md) and install it on the repository.
 2. Store its client ID in the repository variable `RIGHTMODELER_APP_CLIENT_ID` and its private key in the repository secret `RIGHTMODELER_APP_PRIVATE_KEY`.

package/docs/model-routes.md

@@ -0,0 +1,111 @@
+# Model routes
+
+Replay sends each recorded request to cheaper candidate models, and the built-in judge grades every answer against the recorded one. Candidates and the judge each run on a route: an API endpoint with a key, or a plan you are signed in to on this machine through its command-line tool.
+
+## Routes
+
+- `--route api` sends candidate calls to the `--base-url` endpoint with the key named by `--api-key-env`. It is the default when `--base-url` is given, so commands from earlier releases behave as before.
+- `--route claude-login` runs candidate calls through the `claude` CLI you are signed in to, under your Claude plan.
+- `--route codex-login` runs candidate calls through the `codex` CLI you are signed in to, under your ChatGPT plan.
+- `--judge-route api`, `--judge-route claude-login` or `--judge-route codex-login` picks where the built-in judge runs. With `--base-url` and no route flag, the judge uses the API route too.
+
+The judge must come from a vendor other than both the candidate's and the recorded model's. A plan route serves one vendor's models, so a plan `--route` needs an explicit `--judge-route`; without one, rightmodeler stops with `invalid_option`. Candidates from the judge route's own vendor are left out with the warning `judge_vendor_candidates_dropped`, and the run stops before any model call with `no_neutral_judge` when no candidate is left or the recorded model comes from the judge's vendor.
+
+`--base-url`, `--api-key-env` and `--header` configure the API route, so they are refused when both `--route` and `--judge-route` name a plan route. `--detach` and `--modeb-config` are refused when either role uses a plan route: detached replay and Mode B confirmation call models only through an API endpoint. `--evaluator` works with every route; the built-in judge on `--judge-route` grades when the evaluator is unreachable.
+
+```sh
+npx rightmodeler init --traces traces.jsonl --route claude-login --judge-route api --base-url https://openrouter.ai/api/v1 --api-key-env OPENROUTER_API_KEY
+npx rightmodeler init --traces traces.jsonl --base-url https://api.openai.com/v1 --api-key-env OPENAI_API_KEY --catalog-reference https://ai-gateway.vercel.sh/v1/models --judge-route claude-login
+npx rightmodeler init --traces traces.jsonl --route codex-login --judge-route claude-login
+```
+
+## Choosing at the start
+
+Run in a terminal, `init` and `estimate` first ask how to call models, before the trace question and before any stage runs.
+
+- **Your plans:** rightmodeler lists the `claude` and `codex` CLIs on this machine, each with its status and, when it cannot be used, how to fix it. The status comes from each CLI's version and login commands, run with the same API key variables kept away as for a call, and with the variable a saved answer names kept away too. You choose the CLI that replays candidates (choose the vendor your app calls today) and where the judge runs: the other CLI, or an API key for OpenRouter, Vercel AI Gateway or another OpenAI-compatible endpoint. The first time a plan route is chosen, rightmodeler shows what it sends and asks `[y/N]`; any other answer than `y` continues with no route.
+- **An API key:** OpenRouter, Vercel AI Gateway, OpenAI, Anthropic, or another OpenAI-compatible endpoint, then the name of the environment variable that holds the key. The question checks only whether that variable is set, never its value. It accepts only a name made of capital letters, digits and underscores, so a pasted key is refused and never repeated or saved, and it refuses a base URL that carries a user name, a password or a query string.
+
+The answer is saved in the store as `project/setup/model-route.json`: route names, a base URL and a variable name, never a key. The next interactive run shows it as flags and keeps it when you press Enter; type `c` to choose again. `--yes` applies it without asking. The question and the saved answer are skipped with `--output json` or `jsonl`, without a terminal, when `--route`, `--judge-route`, `--base-url`, `--api-key-env` or `--header` is passed, with `init --plan`, and with `--through` before `replay`. Scripts pass the flags the question prints. With `--modeb-config`, only API routes through a multi-vendor endpoint are offered, because Mode B confirmation runs only through an API key. Ctrl-C or Ctrl-D at a question continues with no route: the free stages run and replay stops with `missing_provider_configuration`.
+
+## API routes
+
+| Choice            | `--base-url`                      | Key variable, by default | Also passed                                                                                         |
+| ----------------- | --------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------- |
+| OpenRouter        | `https://openrouter.ai/api/v1`    | `OPENROUTER_API_KEY`     | nothing                                                                                             |
+| Vercel AI Gateway | `https://ai-gateway.vercel.sh/v1` | `AI_GATEWAY_API_KEY`     | nothing                                                                                             |
+| OpenAI            | `https://api.openai.com/v1`       | `OPENAI_API_KEY`         | `--route api --judge-route claude-login --catalog-reference https://ai-gateway.vercel.sh/v1/models` |
+| Anthropic         | `https://api.anthropic.com/v1`    | `ANTHROPIC_API_KEY`      | `--route api --judge-route codex-login --catalog-reference https://ai-gateway.vercel.sh/v1/models`  |
+| Another endpoint  | the URL you type                  | `RIGHTMODELER_API_KEY`   | nothing                                                                                             |
+
+OpenAI's and Anthropic's APIs serve only their own models and list no prices, so the judge runs through the other vendor's CLI signed in on this machine and prices come from the public list; without that CLI, choose a gateway. A variable that is not set yet is still saved: set it in your own shell before replay.
+
+## Use your Claude plan
+
+`--route claude-login` and `--judge-route claude-login` run the `claude` CLI (Claude Code) already installed and signed in on this machine, version 2.1.282 or newer. Rightmodeler runs your own unmodified binary as a child process under the login it already holds. It never reads, stores or forwards a token, and never opens `~/.claude`, the macOS Keychain or a credential file.
+
+Before any model call, rightmodeler runs `claude --version` and `claude auth status`. It accepts only a Claude plan login: signed in, through Anthropic directly, with a claude.ai login or a `claude setup-token` token. Anything else, such as an API key, Amazon Bedrock or Google Vertex, stops with `plan_login_required`.
+
+Each call runs in a fresh, empty temporary directory with no tools, no MCP servers, no settings files, no skills, no auto memory, no session file and one turn: `claude -p --model <id> --system-prompt-file <file> --tools "" --strict-mcp-config --disable-slash-commands --setting-sources "" --no-session-persistence --max-turns 1 --settings {"switchModelsOnFlag":false} --output-format stream-json --verbose`, with the recorded user message on standard input. Because no session file is written, replays never appear in the Claude Code transcripts rightmodeler reads as traces.
+
+When `claude` runs the built-in judge, rightmodeler adds `--json-schema` with the verdict schema and sets `--max-turns 3`. `claude` returns the verdict through its StructuredOutput tool, checks it against the schema and asks the model again when it does not match; rightmodeler reads the verdict from the `structured_output` field. A verdict that still does not match after two corrections, or a call that ends without `structured_output`, counts as a malformed judge answer, as on the API route. Candidate replays get no schema and keep `--max-turns 1`. In rightmodeler's check on `claude` 2.1.283 on 2026-09-26, `--max-turns 3` allowed the first answer and two corrections, and the schema added about 540 input tokens to each Opus 5 judge call; cost estimates use the request's own tokens.
+
+An API key variable would make `claude` bill the key instead of your plan: in non-interactive mode the key is always used when present. Rightmodeler removes every `ANTHROPIC_*` and `OPENAI_*` variable, `CODEX_API_KEY`, the run's `--api-key-env` variable, the other key variables the first-run question offers (`OPENROUTER_API_KEY`, `AI_GATEWAY_API_KEY` and `RIGHTMODELER_API_KEY`), and the variables of a parent Claude Code session from the child's environment, and warns with `plan_route_key_withheld` when `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` was set, naming the variable and never its value. It keeps `CLAUDE_CODE_OAUTH_TOKEN` and `CLAUDE_CONFIG_DIR`. If `claude` still reports that a call would be paid by a key, rightmodeler stops that call at once with `plan_login_required`.
+
+Rightmodeler runs at most `--max-concurrency` `claude` processes at a time (default 2) and stops any still running when it exits. It refuses plan routes when the `CI` environment variable is set: a plan is for your own machine, not for continuous integration.
+
+## Use your ChatGPT plan through Codex
+
+`--route codex-login` and `--judge-route codex-login` run the `codex` CLI already installed and signed in on this machine, version 0.153.3 or newer. As with Claude, rightmodeler runs your own unmodified binary under the login it already holds.
+
+- What runs: `codex exec --json --ephemeral --ignore-user-config --ignore-rules --strict-config --skip-git-repo-check --sandbox read-only`, in a fresh, empty temporary directory, with the recorded system and developer messages in a temporary instructions file (`model_instructions_file`), the recorded user message on standard input, and settings that turn off the shell and other tools, web search, apps, plugins, skills, memories, the injected permission, app and environment context, project instructions and history. Code execution is off too (`features.code_mode_host=false`). A case with no system or developer message runs with `instructions=""` instead of the file, because Codex refuses an empty instructions file and would otherwise add its own default instructions. `--ephemeral` keeps the session file out of `~/.codex/sessions`, where rightmodeler reads Codex traces, and `--ignore-user-config` keeps your `config.toml`, MCP servers and plugins out of the call. `--strict-config` makes a newer `codex` that renamed one of these settings stop with `plan_cli_unavailable` instead of quietly turning a tool back on. When `codex` runs the built-in judge, rightmodeler also writes the verdict schema to that directory and passes `--output-schema`, so Codex's final message is the verdict as JSON.
+- What it never touches: `auth.json`, the keyring or any token. Rightmodeler picks the credential store by name (`file`, else `keyring`) from what `codex login status` reports, and passes that name to each call. `CODEX_API_KEY`, every `OPENAI_*` and every `ANTHROPIC_*` variable, and the key variables listed for Claude above are kept away from `codex`, and `plan_route_key_withheld` names a set `CODEX_API_KEY`, `OPENAI_API_KEY`, `OPENAI_FEDERATION_RULE_ID` or `OPENAI_IDENTITY_TOKEN_FILE`, never its value. `CODEX_HOME` and `CODEX_ACCESS_TOKEN` are passed through unread. A login with an API key, workload identity or Amazon Bedrock stops with `plan_login_required`, because it would not use your ChatGPT plan.
+- What a Codex route measures: Codex adds about 2,000 input tokens of its own context to each call. That includes your global instructions file, `$CODEX_HOME/AGENTS.md` (or `AGENTS.override.md`), which Codex always adds and cannot be told to leave out; rightmodeler checks only that the file exists, never reads it, and warns once with `codex_global_instructions`. Cost estimates use the recorded input tokens, so this context does not change the savings. Codex cannot set temperature or an output limit, and each model runs at its default reasoning effort, as on the API route, which sends none. Codex reports no latency, so the p50 latency of a Codex answer reads n/a.
+- What Codex does not report: which model answered. Rightmodeler records the requested model, and leaves a call out of the evidence when Codex reports that it rerouted the call to another model.
+- Tools: Codex keeps a code tool and a patch tool registered that no setting removes, so a call can include a tool step its output does not show. A call where Codex reports a tool step is left out of the evidence. In rightmodeler's check on `codex-cli` 0.153.3 on 2026-09-25, a prompt asking the model to read a file with its code tool used 13,325 input tokens with code execution on and 3,826 with it off; in both runs the model answered that it could not run anything, did not return the file's contents, and the output showed no tool step.
+- Models: the list comes from `codex debug models` of the same binary that makes the calls, so update Codex to see newer models. GPT-5.5 retires from ChatGPT sign-in on October 14, 2026; it stays on the OpenAI API.
+- Limits: a limit on one model blocks that candidate, or moves the judge to its next model. An account usage limit, a workspace out of credits or a spend cap starts no new call and exits `2` with `plan_usage_limit`, quoting Codex's reset time as Codex wrote it. Rerun after the reset: finished calls are kept. `codex exec` reports no remaining allowance, so rightmodeler cannot warn before the limit.
+- Concurrency: at most `--max-concurrency` `codex` processes run at a time (default 2), each stopped when rightmodeler exits. Plan routes are refused when `CI` is set.
+
+## What a plan route measures
+
+A plan route measures the model inside a coding CLI, not the API request your application makes:
+
+- Claude Code adds its own instructions to every call, even when rightmodeler replaces the system prompt: 380 to 539 input tokens in testing, including the signed-in account's email address and today's date.
+- The CLI cannot set temperature or an output limit, so the recorded values are not applied, and an answer can be longer than on the API route.
+- It sends one user turn. Recorded cases with an earlier assistant or tool turn, or more than one user message, are left out of the replay sample with the warning `plan_route_cases_left_out`.
+- With `claude`, latency is the API time the CLI reports, without its start-up time.
+- With `claude`, rightmodeler checks which model answered on every call, and records a substitution when `claude` answers with another model, takes more than one turn or calls a tool. A judge call is expected to use the StructuredOutput tool and to report up to four turns for the first answer and two corrections, so it counts as a substitution only when `claude` calls another tool or takes more than four turns.
+
+The report's "Model routes" section, and the pull request that `apply` opens, say when a result was measured through a plan.
+
+With `claude`, your recorded prompts are sent to Anthropic under your plan account's data settings. Anthropic's data-usage page says: "We will train new models using data from Free, Pro, and Max accounts when this setting is on (including when you use Claude Code from these accounts)" (`https://code.claude.com/docs/en/data-usage`).
+
+## Costs, the cap and usage limits
+
+Calls through a plan are not billed in dollars: they use your plan's usage allowance; with `claude`, the same 5-hour and weekly limits as your own Claude Code sessions. To budget and compare them, rightmodeler prices each call at API list prices from a public price list, `https://ai-gateway.vercel.sh/v1/models`, read without a key. `--catalog-reference <url-or-path>` replaces that list, including with a local file when the public list is unreachable. `--pricing-file` only overrides the prices of the ids it names.
+
+A call's cost is the recorded request's input tokens plus the output tokens the CLI reports, at list price, and is always marked as an estimate. The CLI's added instructions stay out of it, so savings compare with the recorded calls. The report and `estimate` label these amounts as list-price equivalents and show dollars billed through an API route separately.
+
+Plan routes have no default limit on calls or spend: a run makes as many calls as it needs, and the vendor's own usage limit is the only stop (the run resumes after the reset). `--max-cost-usd` is optional; when you set it on a plan route it caps the list-price equivalent, as a soft cap: the CLI sets no output limit, so a call can cost more than its reservation.
+
+When `claude` reports your plan near its limit, rightmodeler warns once with `plan_usage_warning`, giving the share used and the reset time. When the plan reaches its limit, or further calls would bill usage credits (`credits_required`, or `claude` reporting overage), rightmodeler starts no new call and exits `2` with `plan_usage_limit`, quoting the reset time. Calls already running finish and are kept, so up to `--max-concurrency` calls can still use credits when overage begins. Rerun the same command after the reset: finished replay and judge calls are reused.
+
+Anthropic announced, then paused, a change that would take `claude -p` usage off your plan's limits and onto a monthly credit, then usage credits (`https://support.claude.com/en/articles/15036540-use-the-claude-agent-sdk-with-your-claude-plan`). For now, the page says, `claude -p` still draws from your subscription's usage limits. If the change takes effect, rightmodeler stops at `credits_required` or overage as described above.
+
+Rightmodeler leaves out Claude Fable models, because in non-interactive mode "When a Fable request there would bill to usage credits, Claude Code bills it without asking" (`https://code.claude.com/docs/en/model-config`), and `[1m]` variants, whose 1M context can require usage credits. Models the price list does not price are left out with `plan_model_unpriced`.
+
+## Errors
+
+- `plan_cli_unavailable` (exit `2`): `claude` is missing or older than 2.1.282, `codex` is missing or older than 0.153.3, `CI` is set, the CLI changed an output shape rightmodeler relies on, or `codex` rejected one of rightmodeler's isolation settings.
+- `plan_login_required` (exit `2`): `claude` is not signed in with a Claude plan, `codex` is not signed in with ChatGPT, the CLI would use an API key, or it lost its login during the run.
+- `plan_usage_limit` (exit `2`): the plan reached its usage limit.
+- `no_neutral_judge` and `judge_family_unknown` (exit `2`): no judge from a third vendor is available. See [Exit codes](exit-codes.md).
+
+## Terms
+
+Anthropic's legal page says OAuth authentication "is designed to support ordinary use of Claude Code and other native Anthropic applications", that third-party developers may not "route requests through Free, Pro, or Max plan credentials on behalf of their users", and that this does not "prevent an end user from signing in to the unmodified Claude Code binary with their own Claude subscription" (`https://code.claude.com/docs/en/legal-and-compliance`). The Agent SDK page adds: "Unless previously approved, Anthropic does not allow third party developers to offer claude.ai login or rate limits for their products" (`https://code.claude.com/docs/en/agent-sdk/overview`). Anthropic's Consumer Terms restrict access "through automated or non-human means, whether through a bot, script, or otherwise" except "where we otherwise explicitly permit it" (`https://www.anthropic.com/legal/consumer-terms`).
+
+OpenAI's non-interactive guide says "`codex exec` reuses saved CLI authentication by default." and "API keys are the right default for automation because they are simpler to provision and rotate. Use this path only if you specifically need to run as your Codex account." (`https://learn.chatgpt.com/docs/non-interactive-mode`). OpenAI's pricing page lists "Codex SDK, `codex exec`, and scriptable workflows" as available on Plus, Pro, Business and Enterprise (`https://learn.chatgpt.com/docs/pricing`). OpenAI's Terms of Use list "Automatically or programmatically extract data or Output" among what you may not do (`https://openai.com/policies/terms-of-use/`). OpenAI's CI guide says "Do not use this workflow for public or open-source repositories" about seeding `auth.json` on CI runners (`https://learn.chatgpt.com/docs/auth/ci-cd-auth`).
+
+Refusing plan routes when `CI` is set is rightmodeler's own product choice: a plan route is local and opt-in. Rightmodeler runs on your machine, for you, with your own unmodified `claude` or `codex` and your own login, and never handles a credential. Whether a replay fits your plan's terms is your decision; the API route is always available.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rightmodeler",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Find and prove safe model substitutions from the agent traces you already have.",
   "homepage": "https://www.rightmodeler.com",
   "type": "module",