tare-cli 0.3.1__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tare_cli-0.3.1 → tare_cli-0.4.1}/CONCEPT.md +62 -48
- {tare_cli-0.3.1 → tare_cli-0.4.1}/PKG-INFO +13 -5
- {tare_cli-0.3.1 → tare_cli-0.4.1}/README.md +12 -4
- {tare_cli-0.3.1 → tare_cli-0.4.1}/pyproject.toml +1 -1
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/agents.py +21 -4
- tare_cli-0.4.1/src/tare/answer.py +412 -0
- tare_cli-0.4.1/src/tare/assets/fonts/IBMPlexMono-Regular.woff2 +0 -0
- tare_cli-0.4.1/src/tare/assets/fonts/IBMPlexMono-SemiBold.woff2 +0 -0
- tare_cli-0.4.1/src/tare/assets/fonts/IBMPlexSans-Regular.woff2 +0 -0
- tare_cli-0.4.1/src/tare/assets/fonts/IBMPlexSans-SemiBold.woff2 +0 -0
- tare_cli-0.4.1/src/tare/assets/fonts/OFL.txt +93 -0
- tare_cli-0.4.1/src/tare/assets/logo-dark.svg +260 -0
- tare_cli-0.4.1/src/tare/assets/logo-light.svg +260 -0
- tare_cli-0.4.1/src/tare/assets/tare.css +228 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/calibrate.py +1 -1
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/cli.py +23 -14
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/cliff.py +11 -11
- tare_cli-0.4.1/src/tare/dashboard.html +359 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/dashboard.py +16 -2
- tare_cli-0.4.1/src/tare/overview.html +65 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/probe.py +11 -11
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/room.py +14 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/swap.py +21 -21
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/trail.py +6 -1
- tare_cli-0.4.1/tests/test_agents.py +55 -0
- tare_cli-0.4.1/tests/test_answer.py +117 -0
- tare_cli-0.4.1/tests/test_cli.py +56 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_cliff.py +7 -7
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_dashboard.py +30 -3
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_probe.py +2 -2
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_room.py +29 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_swap.py +35 -7
- tare_cli-0.3.1/src/tare/dashboard.html +0 -269
- tare_cli-0.3.1/src/tare/overview.html +0 -78
- tare_cli-0.3.1/tests/test_agents.py +0 -9
- {tare_cli-0.3.1 → tare_cli-0.4.1}/.gitignore +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/LICENSE +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/__init__.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/capsule.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/fake.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/journal.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/judge.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/recipe.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/src/tare/usage.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/conftest.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_agy.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_calibrate.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_fake.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_judge.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_pi.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_seatbelt.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.1}/tests/test_usage.py +0 -0
|
@@ -7,8 +7,10 @@ until the reading is `tare: 0.00`, unless you pass `--allow-dirty`. It supports
|
|
|
7
7
|
and the Antigravity CLI (`agy`) on Linux and WSL, and Claude Code, Codex and Pi on macOS.
|
|
8
8
|
|
|
9
9
|
This document describes the design and the measurements behind it. The documentation explains the
|
|
10
|
-
words this document uses: room, probe, reading, leak, declared and
|
|
11
|
-
[docs/words.md](docs/words.md), check,
|
|
10
|
+
words this document uses: room, probe, reading, leak, declared and control run in
|
|
11
|
+
[docs/words.md](docs/words.md), check, restart and handover note in [docs/measuring.md](docs/measuring.md).
|
|
12
|
+
The experiment records under `experiments/` call the control run the control run and a restart a
|
|
13
|
+
tail.
|
|
12
14
|
Every claim marked **measured** was observed on WSL2. The section [Measurements](#measurements)
|
|
13
15
|
lists the agent versions and links each result.
|
|
14
16
|
|
|
@@ -72,11 +74,11 @@ your instructions, memories, plugins, hooks and skills. That is not enough.
|
|
|
72
74
|
server keeps the request, which is the context the agent CLI put together, and answers "ok".
|
|
73
75
|
The idea is the blank run from analytical chemistry, which runs the whole procedure without a
|
|
74
76
|
sample to see what the procedure itself brings in.
|
|
75
|
-
- **Control:** the same agent run in your real setup, the
|
|
77
|
+
- **Control:** the same agent run in your real setup, the control run, must show your context. The
|
|
76
78
|
risk was that features that come with the login (connected accounts, account skills) would
|
|
77
79
|
not load against a fake server, so the probe would read clean while a real run is not.
|
|
78
80
|
**Measured:** for Claude Code this risk did not occur. Against the fake model server, the
|
|
79
|
-
|
|
81
|
+
control run still loaded the connected accounts, account skills, plugin skills, hook output,
|
|
80
82
|
global instructions, memories and the email. The room loaded none of them except the email.
|
|
81
83
|
Pointing Claude Code at a custom base URL turns its tool search off and puts every tool schema
|
|
82
84
|
into the request. So the probe and the runs set `ENABLE_TOOL_SEARCH=true` to see the context as
|
|
@@ -96,7 +98,10 @@ A bubblewrap room is a sandbox for one agent run: an empty `/home` with the room
|
|
|
96
98
|
directory at `/home/tare`, a fresh copy of the login, the agent's executable, the project at
|
|
97
99
|
`/work`, and the environment allowlist. `tare claude` and the other agent commands mount your
|
|
98
100
|
project directly, so the agent changes your real files. Calibrate, Swap and Cliff give every run
|
|
99
|
-
its own copy of the project. The room has network access and a working login.
|
|
101
|
+
its own copy of the project. The room has network access and a working login. Where `/etc/resolv.conf` is a link into `/run`
|
|
102
|
+
(systemd-resolved, NetworkManager), the room mounts the one file the link names; without it no host
|
|
103
|
+
name would resolve in the room, and the probe would not notice, because it only talks to the fake
|
|
104
|
+
model server by address. It keeps your setup out of the measurement. It does not protect your machine from a hostile
|
|
100
105
|
agent.
|
|
101
106
|
|
|
102
107
|
A Seatbelt room (macOS) cannot mount anything. It is one directory under `/private/tmp` that
|
|
@@ -134,19 +139,19 @@ the copy does not start inside the renewal window (#126).
|
|
|
134
139
|
### Probe and reading
|
|
135
140
|
|
|
136
141
|
The probe has the three parts of layer 3: the agent in the room against the fake model server,
|
|
137
|
-
the
|
|
142
|
+
the control run, and the reach script. It sorts what it finds into **kinds of context**,
|
|
138
143
|
each checked on its own: instructions, memories, home path, MCP servers (connected accounts count
|
|
139
144
|
here), skills, plugins, extensions, instruction files above the project, the account email, and
|
|
140
145
|
reach.
|
|
141
146
|
|
|
142
147
|
- A kind of context counts only when you have it. tare learns that from your files, for example
|
|
143
148
|
a global instructions file or a skill folder.
|
|
144
|
-
- If you have a kind of context and the
|
|
145
|
-
it. tare then runs the
|
|
149
|
+
- If you have a kind of context and the control run does not show it, the probe could not see
|
|
150
|
+
it. tare then runs the control run once more after 10 seconds. If the second control run still
|
|
146
151
|
does not show that kind of context, the reading is `tare: not proven (blind)`. **Measured:**
|
|
147
|
-
with six runs starting together, a
|
|
152
|
+
with six runs starting together, a control run sent its request before its MCP servers were
|
|
148
153
|
connected.
|
|
149
|
-
- Every probe makes no paid model call, because both the room and the
|
|
154
|
+
- Every probe makes no paid model call, because both the room and the control run talk to the
|
|
150
155
|
fake model server.
|
|
151
156
|
|
|
152
157
|
### Agents
|
|
@@ -154,7 +159,11 @@ reach.
|
|
|
154
159
|
Each agent needs its own way to the fake model server, its own login copy and its own markers:
|
|
155
160
|
lines of your own files that the probe looks for in a request.
|
|
156
161
|
|
|
157
|
-
|
|
162
|
+
tare mounts the agent's own program into the room. Where the name on `PATH` is a mise shim, a link
|
|
163
|
+
to mise that picks the tool by the name it was called as, tare asks mise for the program behind it
|
|
164
|
+
(`mise which`). Mounted in its place, mise would find no installed tool in the room.
|
|
165
|
+
|
|
166
|
+
| Agent | How the room is set up | Way to the fake model server | Kinds of context the control run showed on the measuring machine |
|
|
158
167
|
|---|---|---|---|
|
|
159
168
|
| Claude Code | `CLAUDE_CONFIG_DIR` and `HOME` in the room, a copy of `.credentials.json`, `--strict-mcp-config --setting-sources project,local` | `ANTHROPIC_BASE_URL` | instructions, home path, MCP servers, skills, plugins, email, reach |
|
|
160
169
|
| Codex | `CODEX_HOME` and `HOME` in the room, a copy of `auth.json` only, `--disable remote_plugin` | `-c openai_base_url=<server>/v1` keeps the ChatGPT login | instructions, memories, home path, skills, plugins, custom agents, reach |
|
|
@@ -181,15 +190,15 @@ What each agent needed:
|
|
|
181
190
|
Codex's default model. Pass `-m` when you compare runs.
|
|
182
191
|
- **Pi:** a fresh agent directory is clean on its own and keeps the project's `AGENTS.md`, so the
|
|
183
192
|
room uses no flags. `--no-context-files` would drop the project's `AGENTS.md`. tare finds Pi's
|
|
184
|
-
extension tools by comparing the
|
|
193
|
+
extension tools by comparing the control run's tools with a run that uses `--no-extensions`.
|
|
185
194
|
With standard input left open, Pi waits for piped input and never answers, so tare closes it.
|
|
186
195
|
- **Antigravity CLI:** the fake model server speaks Google's Cloud Code `v1internal` API and
|
|
187
196
|
offers one model. It lists that model in the groups the CLI shows models in
|
|
188
197
|
(`agentModelSorts`), because the CLI rejects a model that no group lists. The
|
|
189
|
-
CLI writes into `~/.gemini` on every run, so its
|
|
198
|
+
CLI writes into `~/.gemini` on every run, so its control run runs with an overlay over
|
|
190
199
|
`~/.gemini`. A user namespace mounts the overlay, and every write goes to a temporary file
|
|
191
200
|
system that disappears after the run. **Measured:** `~/.gemini` kept its fingerprint (paths,
|
|
192
|
-
sizes, modification times) through every
|
|
201
|
+
sizes, modification times) through every control run. The CLI writes each skill's path into
|
|
193
202
|
its skill list, so the probe compares skill lines without the path. Without that, a skill that
|
|
194
203
|
leaked into the room would not match yours, because its path differs.
|
|
195
204
|
- **Instruction files above the project:** Claude Code's one-shot mode, which the probe uses, did
|
|
@@ -199,7 +208,7 @@ What each agent needed:
|
|
|
199
208
|
- **Long prompts:** Linux limits a single argument to about 128 KiB. tare passes the task to
|
|
200
209
|
every agent as an argument, so a longer task fails. Pi could read it from a file with `@file`.
|
|
201
210
|
|
|
202
|
-
### Capsules, trail and
|
|
211
|
+
### Capsules, trail and handover note
|
|
203
212
|
|
|
204
213
|
Calibrate, Swap and Cliff need a run they can stop and continue. During a recorded run, a hook
|
|
205
214
|
archives the workspace after every tool call. Each archive is a **capsule**. The hook comes from
|
|
@@ -213,16 +222,16 @@ A run continues from a capsule in one of two ways:
|
|
|
213
222
|
resumes a session file cut after the step, with `codex exec resume <id>`. Pi resumes with
|
|
214
223
|
`--session`. **Measured:** a resumed Codex session kept its id and finished the remaining plan
|
|
215
224
|
from step 1.
|
|
216
|
-
- **A
|
|
217
|
-
the messages, the tool calls and their results. tare writes a
|
|
225
|
+
- **A handover note.** The **trail** is a record of a run in the same format for every agent: the task,
|
|
226
|
+
the messages, the tool calls and their results. tare writes a handover note from any prefix of it, so
|
|
218
227
|
any agent can continue any run without reading another agent's session format. The Antigravity
|
|
219
228
|
CLI stores its sessions in a format that tare cannot cut, so it always continues with a
|
|
220
|
-
|
|
229
|
+
handover note. Its trail comes from `transcript_full.jsonl`.
|
|
221
230
|
|
|
222
231
|
### Calibrate
|
|
223
232
|
|
|
224
233
|
`tare calibrate` starts each side (an agent with its arguments) several times from a fresh copy
|
|
225
|
-
of the project and reports each side's pass rate with a 95% interval
|
|
234
|
+
of the project and reports each side's pass rate with a likely range (a 95% Wilson interval). A side can have
|
|
226
235
|
its own prompt, so two skills that start with different commands compare in one call. Without a
|
|
227
236
|
check, a run counts as finished when its agent exits with 0, and tare keeps its workspace for
|
|
228
237
|
scoring elsewhere. **Measured:** in a field test that compared two ideation skills, 18 rooms all
|
|
@@ -243,31 +252,32 @@ judge found a real one-cent rounding bug and an overflow.
|
|
|
243
252
|
|
|
244
253
|
### Swap
|
|
245
254
|
|
|
246
|
-
Swap
|
|
247
|
-
|
|
248
|
-
effect (which agent does better), and, where an agent can continue
|
|
249
|
-
|
|
250
|
-
be zero. Swap checks this, and the check is called the
|
|
251
|
-
|
|
255
|
+
Swap stops two runs at several handover points. At each one, each agent continues each run's
|
|
256
|
+
workspace from a handover note. Swap reports a workspace effect (how much the state of the
|
|
257
|
+
workspace matters), an agent effect (which agent does better), and, where an agent can continue
|
|
258
|
+
its own saved session, the note cost. At the handover point 0 both workspaces are the untouched
|
|
259
|
+
project, so the workspace effect there must be zero. Swap checks this, and the check is called the
|
|
260
|
+
start check. When the workspace effect and the agent effect are within 0.1 of each other, Swap
|
|
261
|
+
says they explain the failure about equally.
|
|
252
262
|
|
|
253
|
-
**Measured:** against scripted models with a known truth, the
|
|
254
|
-
passed from the
|
|
255
|
-
to cause.
|
|
263
|
+
**Measured:** against scripted models with a known truth, the start check passed and the blame
|
|
264
|
+
passed from the agent to the workspace between the handovers at 0 and 0.5, as the scripts were
|
|
265
|
+
written to cause.
|
|
256
266
|
|
|
257
267
|
### Cliff
|
|
258
268
|
|
|
259
269
|
Cliff restarts a failed run from its capsules and names the step after which it no longer
|
|
260
|
-
succeeds.
|
|
261
|
-
|
|
262
|
-
and the first bad step, then adds
|
|
263
|
-
the budget is spent. It reports
|
|
264
|
-
|
|
270
|
+
succeeds. Fresh starts are the restarts from the beginning. A step counts as good while its
|
|
271
|
+
restarts pass at least half as often as a fresh start. Cliff halves the range between the last
|
|
272
|
+
good and the first bad step, then adds restarts to the two neighbours until their likely ranges
|
|
273
|
+
separate or the budget is spent. It reports that the task is too hard for the model instead of a
|
|
274
|
+
step only when the upper end of the fresh starts' likely range (a 95% interval) lies below 0.2.
|
|
265
275
|
|
|
266
276
|
**Measured:** against a scripted model whose wrong step is known, Cliff found step 4 with
|
|
267
|
-
separated
|
|
277
|
+
separated likely ranges in 18 restarts. Cliff stays experimental, because neither of two real test
|
|
268
278
|
rounds had a failure that sat in one step (see [Measurements](#measurements)). Swap already
|
|
269
|
-
shows, at three
|
|
270
|
-
Swap shows that rate dropping between two
|
|
279
|
+
shows, at three handover points, how often an agent passes when it continues its own workspace. When a real
|
|
280
|
+
Swap shows that rate dropping between two handover points, Cliff can narrow the drop down to one step. Only
|
|
271
281
|
then does Cliff get new experiments.
|
|
272
282
|
|
|
273
283
|
### Tokens and cost
|
|
@@ -288,30 +298,34 @@ judge read 186.4k tokens in and $0.093 in total, the Codex part estimated from L
|
|
|
288
298
|
|
|
289
299
|
Calibrate, Swap and Cliff write everything into a run directory: a journal (one JSON line per
|
|
290
300
|
event), a `recipe.json`, a `usage.json` with the totals, and the live output of every agent. The dashboard reads only the run
|
|
291
|
-
directory, so `tare watch` shows a finished run the same way as a live one.
|
|
301
|
+
directory, so `tare watch` shows a finished run the same way as a live one. Each run page opens with
|
|
302
|
+
the answer: a sentence from fixed templates per verdict, worked out from the run directory the way
|
|
303
|
+
Calibrate, Swap and Cliff decide, so a run recorded by an older tare reads like a new one. The recipe holds the
|
|
292
304
|
command, a fingerprint of the project and the agent versions. `tare rerun` repeats a run and
|
|
293
305
|
names what has changed since.
|
|
294
306
|
|
|
295
307
|
## Measurements
|
|
296
308
|
|
|
297
309
|
All measurements ran on WSL2 on 2026-10-04 and 2026-10-05. The rows that name macOS ran on
|
|
298
|
-
macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
310
|
+
macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06. The row that names Omarchy ran on native Linux on
|
|
311
|
+
2026-11-16.
|
|
299
312
|
|
|
300
313
|
| What | Agent and version | Result | Source |
|
|
301
314
|
|---|---|---|---|
|
|
302
|
-
|
|
|
303
|
-
|
|
|
304
|
-
|
|
|
305
|
-
|
|
|
315
|
+
| Control run and room, fake model server | Claude Code 2.1.289 | the control run showed every kind of context, the room only the email | [results](experiments/dirty-twin/RESULTS.md) |
|
|
316
|
+
| Control run and room | Codex CLI 0.160.0 | the room read `tare: 0.00` | [results](experiments/dirty-twin-codex/RESULTS.md) |
|
|
317
|
+
| Control run and room | Pi 1.0.2 | the room read `tare: 0.00` without flags | [results](experiments/dirty-twin-pi/RESULTS.md) |
|
|
318
|
+
| Control run and room | Antigravity CLI 1.2.16 | the room read `tare: 0.00` without flags, and `~/.gemini` stayed unchanged | [results](experiments/dirty-twin-agy/RESULTS.md) |
|
|
306
319
|
| Rooms given copies of real settings on purpose | all four | Claude Code 4 leaks, Codex 6, Pi 5, Antigravity CLI 3, each named with its source | the results above |
|
|
307
|
-
| Scripted Cliff and Swap | Claude Code against scripted models | Cliff found step 4. Swap passed the
|
|
320
|
+
| Scripted Cliff and Swap | Claude Code against scripted models | Cliff found step 4. Swap passed the start check and found the blame passing from the agent to the workspace between the handovers at 0 and 0.5 | `experiments/cliff-scripted`, `experiments/swap-scripted` |
|
|
308
321
|
| Real task 1: a data migration | Claude Code with Sonnet and Haiku, Codex | Deleting the source file after a wrong-encoding import was planned as the point of no return. Swap showed that the workspace could still be repaired. Haiku passed about 1 in 14 runs, Codex with gpt-6.1-sol 4 of 4 | [results](experiments/migration/RESULTS.md) |
|
|
309
322
|
| Real task 2: a web page judged by Sonnet | Claude Code with Haiku, Codex with gpt-6.1-sol | Swap confirmed all four predictions of the plan written before the runs. Cliff called a failed run unlucky, and a later rescoring of the same page confirmed it | [results](experiments/html/RESULTS.md) |
|
|
323
|
+
| Native Linux with systemd-resolved: a room that resolves host names | Claude Code 2.1.325, bubblewrap 0.13.1, Omarchy 4.5.0 (Arch Linux) | before the fix the room read `tare: 0.00` and the agent's first request failed with a DNS error; with the resolver file mounted the probe still read `tare: 0.00` and the agent answered one prompt | [#152](https://github.com/a-ratz/tare/issues/152) |
|
|
310
324
|
| Field test: two ideation skills compared | Claude Code 2.1.289 with Opus 5.5 | all 18 rooms read `tare: 0.00`, including a skill that started `claude -p` inside the room | not published |
|
|
311
325
|
| macOS: what a Seatbelt room has to handle | Claude Code 2.1.289, Codex CLI 0.159.0, Pi 0.99.1 | Claude Code's login is in the Keychain. With only the home directory denied, `/var/folders`, `/private/tmp` and the pasteboard stayed readable. Agents' own sandboxes fail inside a room (`sandbox_apply: Operation not permitted`) | comments on epic #85 |
|
|
312
326
|
| macOS: a Seatbelt room with the real login | same | each agent answered one prompt. The room and its login copy were removed afterwards, and the Keychain item was unchanged (digest before and after). `tare probe` read `tare: 0.00` for all three | comments on epic #85 |
|
|
313
|
-
| macOS:
|
|
314
|
-
| macOS: scripted Cliff and Swap, the judge | Claude Code 2.1.290 against scripted models; Microsoft Edge for the judge's screenshot | Cliff found step 4. Swap passed the
|
|
327
|
+
| macOS: control run, Seatbelt room, and parts of the real setup copied into the room | same | every room read `tare: 0.00`. Each copied part that reached the request was named with its source, and nothing under the home directory, under `/var/folders`, in `/private/tmp` or on the pasteboard was reachable from inside. Two kinds of Codex context are invisible to the probe on any system (#109) | [results](experiments/dirty-twin-macos/RESULTS.md) |
|
|
328
|
+
| macOS: scripted Cliff and Swap, the judge | Claude Code 2.1.290 against scripted models; Microsoft Edge for the judge's screenshot | Cliff found step 4. Swap passed the start check and found the blame passing from the agent to the workspace between the handovers at 0 and 0.5. `tare judge` rendered a page and scored it. With a project that turns Claude Code's sandbox on, every Bash call failed in a Seatbelt room until tare turned it off | comments on epic #85 |
|
|
315
329
|
|
|
316
330
|
## Open questions and ideas
|
|
317
331
|
|
|
@@ -321,11 +335,11 @@ macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
|
321
335
|
a fresh room never did. Cached account data is the suspected cause. Not tested.
|
|
322
336
|
- **Claude Code login renewal:** whether a renewal inside the room logs out the real session. Not
|
|
323
337
|
tested, so tare avoids it.
|
|
324
|
-
- **Codex
|
|
338
|
+
- **Codex handover notes:** Codex can run several commands in one tool call, which makes its handover notes
|
|
325
339
|
long.
|
|
326
|
-
- **Cost:** every
|
|
340
|
+
- **Cost:** every restart is a full agent run, so a Cliff search or a Swap costs as much as its restarts.
|
|
327
341
|
- **macOS:** the room is built on Seatbelt, released in 0.3.0 (epic #85). Not done: the
|
|
328
|
-
Antigravity CLI, whose
|
|
342
|
+
Antigravity CLI, whose control run needs Linux's overlay, so tare stops with a message there.
|
|
329
343
|
Not tested: whether a program started through launchd or XPC runs outside the room's profile.
|
|
330
344
|
- **Other probes, not built:** weighing the first request's input tokens before and after adding
|
|
331
345
|
junk to the real setup, and file tripwires that report which secret files a run opened.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tare-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Prove your coding agent starts without your personal setup before you measure it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/a-ratz/tare
|
|
6
6
|
Project-URL: Repository, https://github.com/a-ratz/tare
|
|
@@ -68,7 +68,7 @@ WSL, and with the first three on macOS.
|
|
|
68
68
|
```text
|
|
69
69
|
$ tare probe claude
|
|
70
70
|
tare probe · claude 2.1.289 · /home/me/tare
|
|
71
|
-
control
|
|
71
|
+
control your real setup shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
72
72
|
leaks none
|
|
73
73
|
declared
|
|
74
74
|
email account email subscription login
|
|
@@ -93,13 +93,21 @@ On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). Th
|
|
|
93
93
|
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
94
94
|
every line of the output above.
|
|
95
95
|
|
|
96
|
+
To compare agents, skills or models, `tare calibrate`, `tare swap` and `tare cliff` start many runs
|
|
97
|
+
and open a live dashboard that leads with the answer:
|
|
98
|
+
|
|
99
|
+
<picture>
|
|
100
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-dark.png">
|
|
101
|
+
<img alt="tare's dashboard for a Calibrate run: Claude Code scored higher, 68 against 15 out of 100, with the scores of every run" src="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-light.png">
|
|
102
|
+
</picture>
|
|
103
|
+
|
|
96
104
|
## Features
|
|
97
105
|
|
|
98
106
|
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
99
107
|
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
100
108
|
because an agent asked about its own context can refuse or be wrong.
|
|
101
109
|
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
102
|
-
- **Knows when it cannot see.** A
|
|
110
|
+
- **Knows when it cannot see.** A control run starts the agent in your real setup. If the probe could
|
|
103
111
|
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
104
112
|
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
105
113
|
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
@@ -118,8 +126,8 @@ every line of the output above.
|
|
|
118
126
|
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
119
127
|
from its saved steps and names the step after which it no longer passes.
|
|
120
128
|
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
121
|
-
- **Live dashboard, cost and reruns.**
|
|
122
|
-
and cost, and `tare rerun` repeats a run from its recipe.
|
|
129
|
+
- **Live dashboard, cost and reruns.** The dashboard leads with each run's answer and follows it
|
|
130
|
+
live, each report ends with tokens and cost, and `tare rerun` repeats a run from its recipe.
|
|
123
131
|
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
124
132
|
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
125
133
|
|
|
@@ -43,7 +43,7 @@ WSL, and with the first three on macOS.
|
|
|
43
43
|
```text
|
|
44
44
|
$ tare probe claude
|
|
45
45
|
tare probe · claude 2.1.289 · /home/me/tare
|
|
46
|
-
control
|
|
46
|
+
control your real setup shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
47
47
|
leaks none
|
|
48
48
|
declared
|
|
49
49
|
email account email subscription login
|
|
@@ -68,13 +68,21 @@ On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). Th
|
|
|
68
68
|
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
69
69
|
every line of the output above.
|
|
70
70
|
|
|
71
|
+
To compare agents, skills or models, `tare calibrate`, `tare swap` and `tare cliff` start many runs
|
|
72
|
+
and open a live dashboard that leads with the answer:
|
|
73
|
+
|
|
74
|
+
<picture>
|
|
75
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-dark.png">
|
|
76
|
+
<img alt="tare's dashboard for a Calibrate run: Claude Code scored higher, 68 against 15 out of 100, with the scores of every run" src="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-light.png">
|
|
77
|
+
</picture>
|
|
78
|
+
|
|
71
79
|
## Features
|
|
72
80
|
|
|
73
81
|
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
74
82
|
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
75
83
|
because an agent asked about its own context can refuse or be wrong.
|
|
76
84
|
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
77
|
-
- **Knows when it cannot see.** A
|
|
85
|
+
- **Knows when it cannot see.** A control run starts the agent in your real setup. If the probe could
|
|
78
86
|
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
79
87
|
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
80
88
|
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
@@ -93,8 +101,8 @@ every line of the output above.
|
|
|
93
101
|
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
94
102
|
from its saved steps and names the step after which it no longer passes.
|
|
95
103
|
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
96
|
-
- **Live dashboard, cost and reruns.**
|
|
97
|
-
and cost, and `tare rerun` repeats a run from its recipe.
|
|
104
|
+
- **Live dashboard, cost and reruns.** The dashboard leads with each run's answer and follows it
|
|
105
|
+
live, each report ends with tokens and cost, and `tare rerun` repeats a run from its recipe.
|
|
98
106
|
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
99
107
|
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
100
108
|
|
|
@@ -68,7 +68,24 @@ def _which(name: str) -> Path:
|
|
|
68
68
|
found = shutil.which(name)
|
|
69
69
|
if not found:
|
|
70
70
|
raise TareError(f"{name} is not on PATH")
|
|
71
|
-
|
|
71
|
+
real = Path(found).resolve()
|
|
72
|
+
if real.name == "mise" and Path(found).name != "mise":
|
|
73
|
+
return _behind_mise(name, found, real)
|
|
74
|
+
return real
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _behind_mise(name: str, shim: str, mise: Path) -> Path:
|
|
78
|
+
"""The program behind a mise shim.
|
|
79
|
+
|
|
80
|
+
A mise shim is a link to mise itself, which picks the tool by the name it was called as. Mounted into
|
|
81
|
+
the room in place of the agent, mise finds no installed tool there and tries to fetch one.
|
|
82
|
+
"""
|
|
83
|
+
out = subprocess.run([str(mise), "which", name], capture_output=True, text=True)
|
|
84
|
+
tool = Path(out.stdout.strip()) if out.returncode == 0 and out.stdout.strip() else None
|
|
85
|
+
if tool is None or not tool.is_file():
|
|
86
|
+
raise TareError(f"{name} on PATH is a mise shim ({shim}), and `mise which {name}` did not name the program. "
|
|
87
|
+
f"Put the directory of the program itself first on PATH.")
|
|
88
|
+
return tool.resolve()
|
|
72
89
|
|
|
73
90
|
|
|
74
91
|
def _absolute_links(original: Path, copy: Path):
|
|
@@ -526,7 +543,7 @@ class Pi:
|
|
|
526
543
|
|
|
527
544
|
@contextmanager
|
|
528
545
|
def twin(self, real: Real, fake_url: str):
|
|
529
|
-
"""The
|
|
546
|
+
"""The control run runs on a copy of the user's agent directory; the real one is never written."""
|
|
530
547
|
with tempfile.TemporaryDirectory(prefix="tare-pi-twin-") as copy:
|
|
531
548
|
agent_dir = Path(copy) / "agent"
|
|
532
549
|
shutil.copytree(real.config, agent_dir, symlinks=True, ignore=shutil.ignore_patterns("sessions"))
|
|
@@ -641,7 +658,7 @@ class Antigravity:
|
|
|
641
658
|
|
|
642
659
|
def discover(self) -> Real:
|
|
643
660
|
if sys.platform == "darwin":
|
|
644
|
-
raise TareError("the Antigravity CLI is not supported on macOS yet: its
|
|
661
|
+
raise TareError("the Antigravity CLI is not supported on macOS yet: its control run needs Linux's overlay "
|
|
645
662
|
"over ~/.gemini (epic #85)")
|
|
646
663
|
home = Path.home()
|
|
647
664
|
return Real(home, home / ".gemini" / "antigravity-cli", _which("agy"))
|
|
@@ -679,7 +696,7 @@ class Antigravity:
|
|
|
679
696
|
|
|
680
697
|
@contextmanager
|
|
681
698
|
def twin(self, real: Real, fake_url: str):
|
|
682
|
-
"""The
|
|
699
|
+
"""The control run sees the real setup, but agy writes into ~/.gemini on every run (its
|
|
683
700
|
conversations, caches, onboarding state). So it runs with an overlay over ~/.gemini
|
|
684
701
|
whose writes go to a tmpfs that dies with the run; see twin_argv."""
|
|
685
702
|
mount = tempfile.mkdtemp(prefix="tare-agy-twin-")
|