thwip-cli 1.15.0__tar.gz → 1.16.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/ARCHITECTURE.md +1 -1
  2. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/PKG-INFO +49 -40
  3. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/README.md +48 -39
  4. thwip_cli-1.16.2/docs/benchmark/2026-10-06-final.html +3 -0
  5. thwip_cli-1.16.2/docs/benchmark/2026-10-06-final.json +880 -0
  6. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/verification.md +3 -0
  7. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/pyproject.toml +1 -1
  8. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_config.py +3 -6
  9. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_evals.py +2 -2
  10. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_native_agents.py +6 -8
  11. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_native_print.py +125 -52
  12. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_parity_commands.py +105 -0
  13. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/__init__.py +1 -1
  14. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/native_agent.py +4 -7
  15. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/native_print.py +198 -60
  16. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/cli.py +152 -39
  17. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/config.py +0 -11
  18. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/data/thwip.1 +11 -5
  19. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/evals/runner.py +1 -6
  20. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/evals/tasks.py +17 -11
  21. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/session.py +1 -0
  22. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/uv.lock +1 -1
  23. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/.github/workflows/ci.yml +0 -0
  24. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/.github/workflows/publish.yml +0 -0
  25. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/.gitignore +0 -0
  26. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/LICENSE +0 -0
  27. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/benchmark/2026-10-04-final.html +0 -0
  28. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/benchmark/2026-10-04-final.json +0 -0
  29. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/benchmark/2026-10-04-home-wifi.json +0 -0
  30. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/benchmark/2026-10-04-hotspot.html +0 -0
  31. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/benchmark/2026-10-04-hotspot.json +0 -0
  32. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/docs/handoff-research.md +0 -0
  33. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/install.sh +0 -0
  34. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/scripts/drive_repl.py +0 -0
  35. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/__init__.py +0 -0
  36. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/fake_providers.py +0 -0
  37. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_agents.py +0 -0
  38. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_ai_layers.py +0 -0
  39. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_audit_regressions.py +0 -0
  40. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_catalog.py +0 -0
  41. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_cli.py +0 -0
  42. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_compaction.py +0 -0
  43. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_compatible_streaming.py +0 -0
  44. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_detector.py +0 -0
  45. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_direct_providers_e2e.py +0 -0
  46. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_handoff.py +0 -0
  47. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_memory.py +0 -0
  48. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_native_launcher.py +0 -0
  49. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_native_rpc.py +0 -0
  50. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_prompt.py +0 -0
  51. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_repair_verification.py +0 -0
  52. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_retrieval.py +0 -0
  53. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_session.py +0 -0
  54. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_tools.py +0 -0
  55. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/tests/test_utils.py +0 -0
  56. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/__main__.py +0 -0
  57. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/__init__.py +0 -0
  58. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/base.py +0 -0
  59. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/catalog.py +0 -0
  60. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/chat_messages.py +0 -0
  61. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/claude_agent.py +0 -0
  62. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/deepseek_agent.py +0 -0
  63. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/google_agent.py +0 -0
  64. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/groq_agent.py +0 -0
  65. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/native_common.py +0 -0
  66. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/native_rpc.py +0 -0
  67. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/ollama_agent.py +0 -0
  68. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/openai_agent.py +0 -0
  69. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/agents/openrouter_agent.py +0 -0
  70. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/compaction.py +0 -0
  71. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/data/context_template.md +0 -0
  72. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/detector.py +0 -0
  73. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/endpoints.py +0 -0
  74. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/evals/__init__.py +0 -0
  75. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/evals/__main__.py +0 -0
  76. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/evals/fake_agent.py +0 -0
  77. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/guardrails.py +0 -0
  78. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/handoff.py +0 -0
  79. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/limits.py +0 -0
  80. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/memory.py +0 -0
  81. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/recall.py +0 -0
  82. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/retrieval.py +0 -0
  83. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/shortcuts.py +0 -0
  84. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/structured.py +0 -0
  85. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/theme.py +0 -0
  86. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/tools/__init__.py +0 -0
  87. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/tools/code_runner.py +0 -0
  88. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/tools/file_editor.py +0 -0
  89. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/tools/git_ops.py +0 -0
  90. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/tools/terminal.py +0 -0
  91. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/tracing.py +0 -0
  92. {thwip_cli-1.15.0 → thwip_cli-1.16.2}/thwip/utils.py +0 -0
@@ -94,7 +94,7 @@ sign-in rather than ask for an API key.
94
94
 
95
95
  | File | Lines | What it does |
96
96
  |---|---|---|
97
- | `agents/native_print.py` | 357 | Runs the Claude Code and Antigravity CLIs one turn at a time and reads their streamed output |
97
+ | `agents/native_print.py` | 529 | Runs Claude Code (one process per turn, its permission questions relayed to the prompt) and Antigravity (one process kept open per conversation) and reads their streamed output. thwip adds no restriction of its own |
98
98
  | `agents/native_agent.py` | 288 | Talks to the Codex App Server over JSON-RPC |
99
99
  | `agents/native_common.py` | 129 | Shared helpers: scrubbing output, classifying rate limits, building prompts |
100
100
  | `agents/native_rpc.py` | 86 | The transport that starts the other process and sends messages to it |
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: thwip-cli
3
- Version: 1.15.0
3
+ Version: 1.16.2
4
4
  Summary: Universal coding agent multiplexer: detect, switch, and route between AI coding agents seamlessly
5
5
  Project-URL: Homepage, https://github.com/tanmayhutt/thwip-cli
6
6
  Project-URL: Repository, https://github.com/tanmayhutt/thwip-cli
@@ -155,20 +155,27 @@ transcript, validated on load, shown by `/status`, and cleared by `/new`, `/clea
155
155
  `/compact`, and a project change. If a CLI can no longer continue a session, thwip says
156
156
  so and resends the full conversation to a fresh one.
157
157
 
158
- **Writes.** Native sessions are read-only by default: Claude Code and Antigravity get read-only tools,
159
- and Codex asks you before any write (you answer in thwip). `/writes allow` switches on project-scoped
160
- writes without prompts: Claude Code gets `Edit(./**)` and `Write(./**)`, Antigravity runs in
161
- `accept-edits` mode, and Codex uses a `workspace-write` sandbox. Files outside the project are still
162
- refused by Claude Code and Antigravity. `/writes deny` restores the default; the choice is saved under
163
- `[native] writes` in the config. The evaluation harness runs the file-edit task with writes allowed so
164
- all three CLIs are measured on a real edit.
165
-
166
- Native connections are read-only by default. Codex starts with a read-only sandbox
167
- and asks before operations outside it; approval requests appear in Thwip with the
168
- command or file list and default to denial. Claude Code and the Antigravity CLI run
169
- in their non-interactive print modes, where tools that would need an approval are
170
- declined by the CLI itself. Each turn sends the portable text conversation to a
171
- fresh native session, so native reasoning and tool state do not carry between turns.
158
+ **Permissions.** thwip adds no restrictions of its own. Each CLI runs with the tools, sandbox and
159
+ settings it has on its own, and the questions it would normally ask you are relayed to the thwip
160
+ prompt. Codex runs in its workspace sandbox and asks (through thwip) before anything beyond it.
161
+ Claude Code keeps its own tools and your own settings; anything that would prompt arrives as a
162
+ question in thwip, answered y/N. Antigravity cannot relay questions in headless mode: a tool that
163
+ needs one is declined by the CLI itself and thwip shows its explanation; its own skip-permissions
164
+ flag is the only alternative. `/permissions` prints this summary. The evaluation harness approves the
165
+ CLIs' write questions for the file-edit task so all three are measured on a real edit.
166
+
167
+ **Before a switch.** A switch sends the live conversation to the new CLI. When that transfer would
168
+ fill a large share of the new CLI's window, thwip says how many tokens it is and offers to compact
169
+ first, so the handoff is small. `/handoff` shows the same numbers without switching.
170
+
171
+ **Which project opens.** `--project PATH` always wins. Otherwise the folder you start thwip in is the
172
+ project. Started from your home folder, the last project is the default and the picker offers the
173
+ rest, including New project.
174
+
175
+ Antigravity stays open between turns: starting it costs about 12 seconds of account checks before
176
+ the model is asked, so thwip keeps one process per conversation and sends each new message to it.
177
+ Measured: 41 s for the first turn, 22 s for the next ones. Claude Code and Codex start fast, so
178
+ Claude Code runs one process per turn (sessions resume by ID) and Codex keeps its app server open.
172
179
  Ctrl+C interrupts the current turn and stops the child process; the unanswered
173
180
  message is removed so it can be re-sent or handed to another provider.
174
181
 
@@ -219,7 +226,9 @@ notes are never touched.
219
226
  | Command | Action |
220
227
  |:---|:---|
221
228
  | `/prompt [show|set|reset|save]` | Your standing instructions for every assistant; thwip adds only a neutral note |
222
- | `/writes [allow|deny]` | Let native CLIs edit files inside the project without prompts (default deny) |
229
+ | `/permissions` (`/approvals`) | How each native CLI's own permission questions reach you |
230
+ | `/sessions`, `/continue`, `/init`, `/context`, `/clis` | Aliases for the names Codex, Claude Code and Antigravity use for the same jobs (`/session list`, `/resume`, `/memory init`, `/status`, `/agents`) |
231
+ | `/project [path]` | Pick a known project, create one, or switch to a path; sessions remember their project |
223
232
  | `/recall <words>` | Search earlier turns that compaction archived |
224
233
  | `/trace [n]` | Recent request traces with a per-provider summary |
225
234
  | `/memory` | Show this project's memory file |
@@ -344,30 +353,30 @@ conversation survives summarisation, and `compaction-worker-never-active` checks
344
353
 
345
354
  ## Benchmark
346
355
 
347
- Published at https://thwip.tanmaytiwari.me/#benchmark and stored under `docs/benchmark/`. Thirteen
348
- tasks against the three signed-in CLIs on one machine, 2026-10-04, run twice (home Wi-Fi, then a
349
- phone hotspot) with identical pass/fail results:
350
-
351
- | Provider | Passed (final run) | Mean latency, range across runs |
352
- |:--|:--|:--|
353
- | Claude Code | 13 of 13 | 7.6 to 19.8 s |
354
- | Codex | 10 of 11 | 11.9 to 13.1 s |
355
- | Antigravity | 9 of 11 (one more in an earlier run) | 50.6 to 54.8 s |
356
-
357
- Three full runs on 2026-10-04 are stored (`home-wifi`, `hotspot`, `final`). Pass counts were stable
358
- for Claude Code and Codex; Antigravity varies by one depending on whether it reaches for a tool that
359
- its print mode then denies.
360
-
361
- What failed, honestly. Codex reads a file outside the project with its own shell tools: its read-only
362
- sandbox permits reads anywhere and auto-approves them, and no setting thwip can pass changes that.
363
- Claude Code used to do the same; since v1.14.0 thwip confines its read tools to the project with
364
- permission patterns (`Read(./**)`, `Glob(./**)`, `Grep(./**)`), verified live, so the path guard now
365
- holds for direct API models and Claude Code. Antigravity returned empty answers twice: its print mode
366
- auto-denies any tool that needs a permission prompt and returns nothing, explaining itself only on
367
- stderr; thwip surfaces that explanation. Antigravity's latency is its own per-turn work, not the
368
- network: 54 s on home Wi-Fi, 51 s on the hotspot; a kept-open process was tried and cut a turn to
369
- about 20 s but still carried 18 s of overhead and hung on the third turn, so it was not adopted. The
370
- file-edit task runs with `/writes allow` and all three CLIs edit the file correctly.
356
+ The evaluation harness (`python -m thwip.evals --provider all --html`) runs the thirteen
357
+ tasks against the three signed-in CLIs on one machine. The published run is from 2026-10-06
358
+ (home Wi-Fi); the three 2026-10-04 runs (`home-wifi`, `hotspot`, `final`) are kept next to it.
359
+ Tasks that do not apply to a provider are marked n/a and excluded from the pass count.
360
+
361
+ | Provider | Passed (2026-10-06) | Mean latency |
362
+ | --- | --- | --- |
363
+ | Claude Code | 12 of 12 (+1 n/a) | 8.2 s |
364
+ | Codex | 10 of 10 (+1 n/a) | 13.0 s |
365
+ | Antigravity | 8 of 9 (+2 n/a) | 70.0 s |
366
+
367
+ Each task is a fresh conversation, so Antigravity pays its 12 s of account checks on every task
368
+ here; through the REPL, where the process stays open, later turns take about 22 s. Antigravity's
369
+ one failure is the honesty task: it reached for a tool it could not get permission for in headless
370
+ mode and returned nothing. One Antigravity task hit a connection reset on the home network and was
371
+ rerun on its own; the stored report holds the rerun.
372
+
373
+ What the results mean. The path-guard task applies to thwip's own tools only: since v1.16.0 thwip
374
+ deliberately adds no restriction to a native CLI, so that task is marked not applicable for all three.
375
+ Antigravity returned empty answers twice: its headless mode declines any tool that needs a permission
376
+ prompt and returns nothing, explaining itself only on stderr; thwip surfaces that explanation.
377
+ Antigravity's latency is its own per-turn work, not the network: 54 s on home Wi-Fi, 51 s on the
378
+ hotspot with one process per turn; since v1.16.0 the process is kept open and later turns take about
379
+ 22 s. The file-edit task approves the CLIs' own write questions and all three edit the file correctly.
371
380
 
372
381
  ## Recall, structured outputs, tracing, guardrails
373
382
 
@@ -118,20 +118,27 @@ transcript, validated on load, shown by `/status`, and cleared by `/new`, `/clea
118
118
  `/compact`, and a project change. If a CLI can no longer continue a session, thwip says
119
119
  so and resends the full conversation to a fresh one.
120
120
 
121
- **Writes.** Native sessions are read-only by default: Claude Code and Antigravity get read-only tools,
122
- and Codex asks you before any write (you answer in thwip). `/writes allow` switches on project-scoped
123
- writes without prompts: Claude Code gets `Edit(./**)` and `Write(./**)`, Antigravity runs in
124
- `accept-edits` mode, and Codex uses a `workspace-write` sandbox. Files outside the project are still
125
- refused by Claude Code and Antigravity. `/writes deny` restores the default; the choice is saved under
126
- `[native] writes` in the config. The evaluation harness runs the file-edit task with writes allowed so
127
- all three CLIs are measured on a real edit.
128
-
129
- Native connections are read-only by default. Codex starts with a read-only sandbox
130
- and asks before operations outside it; approval requests appear in Thwip with the
131
- command or file list and default to denial. Claude Code and the Antigravity CLI run
132
- in their non-interactive print modes, where tools that would need an approval are
133
- declined by the CLI itself. Each turn sends the portable text conversation to a
134
- fresh native session, so native reasoning and tool state do not carry between turns.
121
+ **Permissions.** thwip adds no restrictions of its own. Each CLI runs with the tools, sandbox and
122
+ settings it has on its own, and the questions it would normally ask you are relayed to the thwip
123
+ prompt. Codex runs in its workspace sandbox and asks (through thwip) before anything beyond it.
124
+ Claude Code keeps its own tools and your own settings; anything that would prompt arrives as a
125
+ question in thwip, answered y/N. Antigravity cannot relay questions in headless mode: a tool that
126
+ needs one is declined by the CLI itself and thwip shows its explanation; its own skip-permissions
127
+ flag is the only alternative. `/permissions` prints this summary. The evaluation harness approves the
128
+ CLIs' write questions for the file-edit task so all three are measured on a real edit.
129
+
130
+ **Before a switch.** A switch sends the live conversation to the new CLI. When that transfer would
131
+ fill a large share of the new CLI's window, thwip says how many tokens it is and offers to compact
132
+ first, so the handoff is small. `/handoff` shows the same numbers without switching.
133
+
134
+ **Which project opens.** `--project PATH` always wins. Otherwise the folder you start thwip in is the
135
+ project. Started from your home folder, the last project is the default and the picker offers the
136
+ rest, including New project.
137
+
138
+ Antigravity stays open between turns: starting it costs about 12 seconds of account checks before
139
+ the model is asked, so thwip keeps one process per conversation and sends each new message to it.
140
+ Measured: 41 s for the first turn, 22 s for the next ones. Claude Code and Codex start fast, so
141
+ Claude Code runs one process per turn (sessions resume by ID) and Codex keeps its app server open.
135
142
  Ctrl+C interrupts the current turn and stops the child process; the unanswered
136
143
  message is removed so it can be re-sent or handed to another provider.
137
144
 
@@ -182,7 +189,9 @@ notes are never touched.
182
189
  | Command | Action |
183
190
  |:---|:---|
184
191
  | `/prompt [show|set|reset|save]` | Your standing instructions for every assistant; thwip adds only a neutral note |
185
- | `/writes [allow|deny]` | Let native CLIs edit files inside the project without prompts (default deny) |
192
+ | `/permissions` (`/approvals`) | How each native CLI's own permission questions reach you |
193
+ | `/sessions`, `/continue`, `/init`, `/context`, `/clis` | Aliases for the names Codex, Claude Code and Antigravity use for the same jobs (`/session list`, `/resume`, `/memory init`, `/status`, `/agents`) |
194
+ | `/project [path]` | Pick a known project, create one, or switch to a path; sessions remember their project |
186
195
  | `/recall <words>` | Search earlier turns that compaction archived |
187
196
  | `/trace [n]` | Recent request traces with a per-provider summary |
188
197
  | `/memory` | Show this project's memory file |
@@ -307,30 +316,30 @@ conversation survives summarisation, and `compaction-worker-never-active` checks
307
316
 
308
317
  ## Benchmark
309
318
 
310
- Published at https://thwip.tanmaytiwari.me/#benchmark and stored under `docs/benchmark/`. Thirteen
311
- tasks against the three signed-in CLIs on one machine, 2026-10-04, run twice (home Wi-Fi, then a
312
- phone hotspot) with identical pass/fail results:
313
-
314
- | Provider | Passed (final run) | Mean latency, range across runs |
315
- |:--|:--|:--|
316
- | Claude Code | 13 of 13 | 7.6 to 19.8 s |
317
- | Codex | 10 of 11 | 11.9 to 13.1 s |
318
- | Antigravity | 9 of 11 (one more in an earlier run) | 50.6 to 54.8 s |
319
-
320
- Three full runs on 2026-10-04 are stored (`home-wifi`, `hotspot`, `final`). Pass counts were stable
321
- for Claude Code and Codex; Antigravity varies by one depending on whether it reaches for a tool that
322
- its print mode then denies.
323
-
324
- What failed, honestly. Codex reads a file outside the project with its own shell tools: its read-only
325
- sandbox permits reads anywhere and auto-approves them, and no setting thwip can pass changes that.
326
- Claude Code used to do the same; since v1.14.0 thwip confines its read tools to the project with
327
- permission patterns (`Read(./**)`, `Glob(./**)`, `Grep(./**)`), verified live, so the path guard now
328
- holds for direct API models and Claude Code. Antigravity returned empty answers twice: its print mode
329
- auto-denies any tool that needs a permission prompt and returns nothing, explaining itself only on
330
- stderr; thwip surfaces that explanation. Antigravity's latency is its own per-turn work, not the
331
- network: 54 s on home Wi-Fi, 51 s on the hotspot; a kept-open process was tried and cut a turn to
332
- about 20 s but still carried 18 s of overhead and hung on the third turn, so it was not adopted. The
333
- file-edit task runs with `/writes allow` and all three CLIs edit the file correctly.
319
+ The evaluation harness (`python -m thwip.evals --provider all --html`) runs the thirteen
320
+ tasks against the three signed-in CLIs on one machine. The published run is from 2026-10-06
321
+ (home Wi-Fi); the three 2026-10-04 runs (`home-wifi`, `hotspot`, `final`) are kept next to it.
322
+ Tasks that do not apply to a provider are marked n/a and excluded from the pass count.
323
+
324
+ | Provider | Passed (2026-10-06) | Mean latency |
325
+ | --- | --- | --- |
326
+ | Claude Code | 12 of 12 (+1 n/a) | 8.2 s |
327
+ | Codex | 10 of 10 (+1 n/a) | 13.0 s |
328
+ | Antigravity | 8 of 9 (+2 n/a) | 70.0 s |
329
+
330
+ Each task is a fresh conversation, so Antigravity pays its 12 s of account checks on every task
331
+ here; through the REPL, where the process stays open, later turns take about 22 s. Antigravity's
332
+ one failure is the honesty task: it reached for a tool it could not get permission for in headless
333
+ mode and returned nothing. One Antigravity task hit a connection reset on the home network and was
334
+ rerun on its own; the stored report holds the rerun.
335
+
336
+ What the results mean. The path-guard task applies to thwip's own tools only: since v1.16.0 thwip
337
+ deliberately adds no restriction to a native CLI, so that task is marked not applicable for all three.
338
+ Antigravity returned empty answers twice: its headless mode declines any tool that needs a permission
339
+ prompt and returns nothing, explaining itself only on stderr; thwip surfaces that explanation.
340
+ Antigravity's latency is its own per-turn work, not the network: 54 s on home Wi-Fi, 51 s on the
341
+ hotspot with one process per turn; since v1.16.0 the process is kept open and later turns take about
342
+ 22 s. The file-edit task approves the CLIs' own write questions and all three edit the file correctly.
334
343
 
335
344
  ## Recall, structured outputs, tracing, guardrails
336
345
 
@@ -0,0 +1,3 @@
1
+ <p class="bench-meta">Measured 2026-10-06T13:20:46 on the maintainer's machine with the signed-in CLIs. 35 runs.</p>
2
+ <table class="commands-table"><thead><tr><th>Provider</th><th>Passed</th><th>Mean latency s</th><th>Tool calls</th><th>Malformed</th><th>Errors</th></tr></thead><tbody><tr><td>claude</td><td>12/12 (1 not applicable)</td><td>8.2</td><td>0</td><td>0</td><td>0</td></tr><tr><td>google</td><td>8/9 (2 not applicable)</td><td>70.0</td><td>0</td><td>0</td><td>0</td></tr><tr><td>openai</td><td>10/10 (1 not applicable)</td><td>13.0</td><td>0</td><td>0</td><td>0</td></tr></tbody></table>
3
+ <details class="bench-detail"><summary>Every run</summary><table class="commands-table"><thead><tr><th>Task</th><th>Provider</th><th>Result</th><th>Latency s</th><th>Input tokens</th><th>Note</th></tr></thead><tbody><tr><td>tool-schemas-consistent</td><td>claude</td><td>pass</td><td>0.0</td><td>0</td><td>8 tools consistent across both shapes; 4 invalid calls rejected</td></tr><tr><td>compaction-worker-never-active</td><td>claude</td><td>pass</td><td>0.0</td><td>0</td><td>active provider skipped; chain order honored; last resort handled</td></tr><tr><td>contract-stream-and-done</td><td>claude</td><td>pass</td><td>2.2</td><td>22,068</td><td>3 events, usage 22068/13</td></tr><tr><td>exact-single-word</td><td>claude</td><td>pass</td><td>2.9</td><td>22,071</td><td>answer was &#x27;pong&#x27;</td></tr><tr><td>tool-read-file</td><td>claude</td><td>pass</td><td>4.4</td><td>46,503</td><td>native CLI used its own tools; answer &#x27;PLUM&#x27;</td></tr><tr><td>tool-path-containment</td><td>claude</td><td>n/a</td><td>12.2</td><td>48,076</td><td>skipped: thwip adds no restrictions to native CLIs by design; the path guard covers thwip&#x27;</td></tr><tr><td>memory-deep-recall</td><td>claude</td><td>pass</td><td>2.3</td><td>22,309</td><td>deep recall: answer &#x27;THURSDAY&#x27;</td></tr><tr><td>handoff-recall</td><td>claude</td><td>pass</td><td>4.2</td><td>22,229</td><td>handoff recall: answer &#x27;MAGENTA&#x27;</td></tr><tr><td>compaction-recall</td><td>claude</td><td>pass</td><td>8.2</td><td>22,534</td><td>compaction recall: answer &#x27;OLIVE&#x27;</td></tr><tr><td>tool-edit-file</td><td>claude</td><td>pass</td><td>7.0</td><td>74,367</td><td>edited via the CLI with its own tools (write questions approved by the harness)</td></tr><tr><td>missing-file-honesty</td><td>claude</td><td>pass</td><td>34.6</td><td>74,488</td><td>reported missing: &quot;I&#x27;ll read the release notes file now.The file isn&#x27;t at the p&quot;</td></tr><tr><td>archive-recall</td><td>claude</td><td>pass</td><td>18.9</td><td>22,814</td><td>archive recall: answer &#x27;SAFFRON&#x27;</td></tr><tr><td>structured-summary</td><td>claude</td><td>pass</td><td>10.1</td><td>0</td><td>valid JSON with 3 context and 1 decision items</td></tr><tr><td>contract-stream-and-done</td><td>google</td><td>pass</td><td>41.3</td><td>16,680</td><td>3 events, usage 16680/106</td></tr><tr><td>exact-single-word</td><td>google</td><td>pass</td><td>40.3</td><td>16,681</td><td>answer was &#x27;pong&#x27;</td></tr><tr><td>tool-read-file</td><td>google</td><td>pass</td><td>54.0</td><td>33,825</td><td>native CLI used its own tools; answer &#x27;PLUM&#x27;</td></tr><tr><td>tool-path-containment</td><td>google</td><td>n/a</td><td>85.9</td><td>4,962</td><td>skipped: thwip adds no restrictions to native CLIs by design; the path guard covers thwip&#x27;</td></tr><tr><td>memory-deep-recall</td><td>google</td><td>pass</td><td>71.1</td><td>16,856</td><td>deep recall: answer &#x27;THURSDAY&#x27;</td></tr><tr><td>handoff-recall</td><td>google</td><td>pass</td><td>41.8</td><td>16,787</td><td>handoff recall: answer &#x27;MAGENTA&#x27;</td></tr><tr><td>compaction-recall</td><td>google</td><td>pass</td><td>98.7</td><td>16,888</td><td>compaction recall: answer &#x27;OLIVE&#x27;</td></tr><tr><td>tool-edit-file</td><td>google</td><td>n/a</td><td>47.4</td><td>33,884</td><td>skipped: the CLI cannot ask for write permission in headless mode and said so</td></tr><tr><td>missing-file-honesty</td><td>google</td><td>fail</td><td>47.8</td><td>21,619</td><td>answer did not clearly say the file is missing: &#x27;[Antigravity CLI produced no answer: it n</td></tr><tr><td>archive-recall</td><td>google</td><td>pass</td><td>88.0</td><td>16,987</td><td>archive recall: answer &#x27;SAFFRON&#x27;</td></tr><tr><td>structured-summary</td><td>google</td><td>pass</td><td>153.9</td><td>0</td><td>valid JSON with 2 context and 1 decision items</td></tr><tr><td>contract-stream-and-done</td><td>openai</td><td>pass</td><td>12.7</td><td>18,439</td><td>3 events, usage 18439/6</td></tr><tr><td>exact-single-word</td><td>openai</td><td>pass</td><td>7.3</td><td>17,746</td><td>answer was &#x27;pong&#x27;</td></tr><tr><td>tool-read-file</td><td>openai</td><td>pass</td><td>11.5</td><td>17,907</td><td>native CLI used its own tools; answer &#x27;PLUM&#x27;</td></tr><tr><td>tool-path-containment</td><td>openai</td><td>n/a</td><td>14.0</td><td>17,974</td><td>skipped: thwip adds no restrictions to native CLIs by design; the path guard covers thwip&#x27;</td></tr><tr><td>memory-deep-recall</td><td>openai</td><td>pass</td><td>9.0</td><td>17,906</td><td>deep recall: answer &#x27;THURSDAY&#x27;</td></tr><tr><td>handoff-recall</td><td>openai</td><td>pass</td><td>7.3</td><td>17,832</td><td>handoff recall: answer &#x27;MAGENTA&#x27;</td></tr><tr><td>compaction-recall</td><td>openai</td><td>pass</td><td>17.7</td><td>18,638</td><td>compaction recall: answer &#x27;OLIVE&#x27;</td></tr><tr><td>tool-edit-file</td><td>openai</td><td>pass</td><td>21.7</td><td>19,827</td><td>edited via the CLI with its own tools (write questions approved by the harness)</td></tr><tr><td>missing-file-honesty</td><td>openai</td><td>pass</td><td>19.6</td><td>18,304</td><td>reported missing: &#x27;I’ll inspect the release notes and summarize the newest entr&#x27;</td></tr><tr><td>archive-recall</td><td>openai</td><td>pass</td><td>14.8</td><td>18,733</td><td>archive recall: answer &#x27;SAFFRON&#x27;</td></tr><tr><td>structured-summary</td><td>openai</td><td>pass</td><td>7.9</td><td>0</td><td>valid JSON with 2 context and 1 decision items</td></tr></tbody></table></details>