tare-cli 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tare_cli-0.3.1 → tare_cli-0.4.0}/CONCEPT.md +51 -46
- {tare_cli-0.3.1 → tare_cli-0.4.0}/PKG-INFO +13 -5
- {tare_cli-0.3.1 → tare_cli-0.4.0}/README.md +12 -4
- {tare_cli-0.3.1 → tare_cli-0.4.0}/pyproject.toml +1 -1
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/agents.py +3 -3
- tare_cli-0.4.0/src/tare/answer.py +412 -0
- tare_cli-0.4.0/src/tare/assets/fonts/IBMPlexMono-Regular.woff2 +0 -0
- tare_cli-0.4.0/src/tare/assets/fonts/IBMPlexMono-SemiBold.woff2 +0 -0
- tare_cli-0.4.0/src/tare/assets/fonts/IBMPlexSans-Regular.woff2 +0 -0
- tare_cli-0.4.0/src/tare/assets/fonts/IBMPlexSans-SemiBold.woff2 +0 -0
- tare_cli-0.4.0/src/tare/assets/fonts/OFL.txt +93 -0
- tare_cli-0.4.0/src/tare/assets/logo-dark.svg +260 -0
- tare_cli-0.4.0/src/tare/assets/logo-light.svg +260 -0
- tare_cli-0.4.0/src/tare/assets/tare.css +228 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/calibrate.py +1 -1
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/cli.py +23 -14
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/cliff.py +11 -11
- tare_cli-0.4.0/src/tare/dashboard.html +359 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/dashboard.py +16 -2
- tare_cli-0.4.0/src/tare/overview.html +65 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/probe.py +11 -11
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/swap.py +21 -21
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/trail.py +6 -1
- tare_cli-0.4.0/tests/test_answer.py +117 -0
- tare_cli-0.4.0/tests/test_cli.py +56 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_cliff.py +7 -7
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_dashboard.py +30 -3
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_probe.py +2 -2
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_swap.py +35 -7
- tare_cli-0.3.1/src/tare/dashboard.html +0 -269
- tare_cli-0.3.1/src/tare/overview.html +0 -78
- {tare_cli-0.3.1 → tare_cli-0.4.0}/.gitignore +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/LICENSE +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/__init__.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/capsule.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/fake.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/journal.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/judge.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/recipe.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/room.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/src/tare/usage.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/conftest.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_agents.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_agy.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_calibrate.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_fake.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_judge.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_pi.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_room.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_seatbelt.py +0 -0
- {tare_cli-0.3.1 → tare_cli-0.4.0}/tests/test_usage.py +0 -0
|
@@ -7,8 +7,10 @@ until the reading is `tare: 0.00`, unless you pass `--allow-dirty`. It supports
|
|
|
7
7
|
and the Antigravity CLI (`agy`) on Linux and WSL, and Claude Code, Codex and Pi on macOS.
|
|
8
8
|
|
|
9
9
|
This document describes the design and the measurements behind it. The documentation explains the
|
|
10
|
-
words this document uses: room, probe, reading, leak, declared and
|
|
11
|
-
[docs/words.md](docs/words.md), check,
|
|
10
|
+
words this document uses: room, probe, reading, leak, declared and control run in
|
|
11
|
+
[docs/words.md](docs/words.md), check, restart and handover note in [docs/measuring.md](docs/measuring.md).
|
|
12
|
+
The experiment records under `experiments/` call the control run the control run and a restart a
|
|
13
|
+
tail.
|
|
12
14
|
Every claim marked **measured** was observed on WSL2. The section [Measurements](#measurements)
|
|
13
15
|
lists the agent versions and links each result.
|
|
14
16
|
|
|
@@ -72,11 +74,11 @@ your instructions, memories, plugins, hooks and skills. That is not enough.
|
|
|
72
74
|
server keeps the request, which is the context the agent CLI put together, and answers "ok".
|
|
73
75
|
The idea is the blank run from analytical chemistry, which runs the whole procedure without a
|
|
74
76
|
sample to see what the procedure itself brings in.
|
|
75
|
-
- **Control:** the same agent run in your real setup, the
|
|
77
|
+
- **Control:** the same agent run in your real setup, the control run, must show your context. The
|
|
76
78
|
risk was that features that come with the login (connected accounts, account skills) would
|
|
77
79
|
not load against a fake server, so the probe would read clean while a real run is not.
|
|
78
80
|
**Measured:** for Claude Code this risk did not occur. Against the fake model server, the
|
|
79
|
-
|
|
81
|
+
control run still loaded the connected accounts, account skills, plugin skills, hook output,
|
|
80
82
|
global instructions, memories and the email. The room loaded none of them except the email.
|
|
81
83
|
Pointing Claude Code at a custom base URL turns its tool search off and puts every tool schema
|
|
82
84
|
into the request. So the probe and the runs set `ENABLE_TOOL_SEARCH=true` to see the context as
|
|
@@ -134,19 +136,19 @@ the copy does not start inside the renewal window (#126).
|
|
|
134
136
|
### Probe and reading
|
|
135
137
|
|
|
136
138
|
The probe has the three parts of layer 3: the agent in the room against the fake model server,
|
|
137
|
-
the
|
|
139
|
+
the control run, and the reach script. It sorts what it finds into **kinds of context**,
|
|
138
140
|
each checked on its own: instructions, memories, home path, MCP servers (connected accounts count
|
|
139
141
|
here), skills, plugins, extensions, instruction files above the project, the account email, and
|
|
140
142
|
reach.
|
|
141
143
|
|
|
142
144
|
- A kind of context counts only when you have it. tare learns that from your files, for example
|
|
143
145
|
a global instructions file or a skill folder.
|
|
144
|
-
- If you have a kind of context and the
|
|
145
|
-
it. tare then runs the
|
|
146
|
+
- If you have a kind of context and the control run does not show it, the probe could not see
|
|
147
|
+
it. tare then runs the control run once more after 10 seconds. If the second control run still
|
|
146
148
|
does not show that kind of context, the reading is `tare: not proven (blind)`. **Measured:**
|
|
147
|
-
with six runs starting together, a
|
|
149
|
+
with six runs starting together, a control run sent its request before its MCP servers were
|
|
148
150
|
connected.
|
|
149
|
-
- Every probe makes no paid model call, because both the room and the
|
|
151
|
+
- Every probe makes no paid model call, because both the room and the control run talk to the
|
|
150
152
|
fake model server.
|
|
151
153
|
|
|
152
154
|
### Agents
|
|
@@ -154,7 +156,7 @@ reach.
|
|
|
154
156
|
Each agent needs its own way to the fake model server, its own login copy and its own markers:
|
|
155
157
|
lines of your own files that the probe looks for in a request.
|
|
156
158
|
|
|
157
|
-
| Agent | How the room is set up | Way to the fake model server | Kinds of context the
|
|
159
|
+
| Agent | How the room is set up | Way to the fake model server | Kinds of context the control run showed on the measuring machine |
|
|
158
160
|
|---|---|---|---|
|
|
159
161
|
| Claude Code | `CLAUDE_CONFIG_DIR` and `HOME` in the room, a copy of `.credentials.json`, `--strict-mcp-config --setting-sources project,local` | `ANTHROPIC_BASE_URL` | instructions, home path, MCP servers, skills, plugins, email, reach |
|
|
160
162
|
| Codex | `CODEX_HOME` and `HOME` in the room, a copy of `auth.json` only, `--disable remote_plugin` | `-c openai_base_url=<server>/v1` keeps the ChatGPT login | instructions, memories, home path, skills, plugins, custom agents, reach |
|
|
@@ -181,15 +183,15 @@ What each agent needed:
|
|
|
181
183
|
Codex's default model. Pass `-m` when you compare runs.
|
|
182
184
|
- **Pi:** a fresh agent directory is clean on its own and keeps the project's `AGENTS.md`, so the
|
|
183
185
|
room uses no flags. `--no-context-files` would drop the project's `AGENTS.md`. tare finds Pi's
|
|
184
|
-
extension tools by comparing the
|
|
186
|
+
extension tools by comparing the control run's tools with a run that uses `--no-extensions`.
|
|
185
187
|
With standard input left open, Pi waits for piped input and never answers, so tare closes it.
|
|
186
188
|
- **Antigravity CLI:** the fake model server speaks Google's Cloud Code `v1internal` API and
|
|
187
189
|
offers one model. It lists that model in the groups the CLI shows models in
|
|
188
190
|
(`agentModelSorts`), because the CLI rejects a model that no group lists. The
|
|
189
|
-
CLI writes into `~/.gemini` on every run, so its
|
|
191
|
+
CLI writes into `~/.gemini` on every run, so its control run runs with an overlay over
|
|
190
192
|
`~/.gemini`. A user namespace mounts the overlay, and every write goes to a temporary file
|
|
191
193
|
system that disappears after the run. **Measured:** `~/.gemini` kept its fingerprint (paths,
|
|
192
|
-
sizes, modification times) through every
|
|
194
|
+
sizes, modification times) through every control run. The CLI writes each skill's path into
|
|
193
195
|
its skill list, so the probe compares skill lines without the path. Without that, a skill that
|
|
194
196
|
leaked into the room would not match yours, because its path differs.
|
|
195
197
|
- **Instruction files above the project:** Claude Code's one-shot mode, which the probe uses, did
|
|
@@ -199,7 +201,7 @@ What each agent needed:
|
|
|
199
201
|
- **Long prompts:** Linux limits a single argument to about 128 KiB. tare passes the task to
|
|
200
202
|
every agent as an argument, so a longer task fails. Pi could read it from a file with `@file`.
|
|
201
203
|
|
|
202
|
-
### Capsules, trail and
|
|
204
|
+
### Capsules, trail and handover note
|
|
203
205
|
|
|
204
206
|
Calibrate, Swap and Cliff need a run they can stop and continue. During a recorded run, a hook
|
|
205
207
|
archives the workspace after every tool call. Each archive is a **capsule**. The hook comes from
|
|
@@ -213,16 +215,16 @@ A run continues from a capsule in one of two ways:
|
|
|
213
215
|
resumes a session file cut after the step, with `codex exec resume <id>`. Pi resumes with
|
|
214
216
|
`--session`. **Measured:** a resumed Codex session kept its id and finished the remaining plan
|
|
215
217
|
from step 1.
|
|
216
|
-
- **A
|
|
217
|
-
the messages, the tool calls and their results. tare writes a
|
|
218
|
+
- **A handover note.** The **trail** is a record of a run in the same format for every agent: the task,
|
|
219
|
+
the messages, the tool calls and their results. tare writes a handover note from any prefix of it, so
|
|
218
220
|
any agent can continue any run without reading another agent's session format. The Antigravity
|
|
219
221
|
CLI stores its sessions in a format that tare cannot cut, so it always continues with a
|
|
220
|
-
|
|
222
|
+
handover note. Its trail comes from `transcript_full.jsonl`.
|
|
221
223
|
|
|
222
224
|
### Calibrate
|
|
223
225
|
|
|
224
226
|
`tare calibrate` starts each side (an agent with its arguments) several times from a fresh copy
|
|
225
|
-
of the project and reports each side's pass rate with a 95% interval
|
|
227
|
+
of the project and reports each side's pass rate with a likely range (a 95% Wilson interval). A side can have
|
|
226
228
|
its own prompt, so two skills that start with different commands compare in one call. Without a
|
|
227
229
|
check, a run counts as finished when its agent exits with 0, and tare keeps its workspace for
|
|
228
230
|
scoring elsewhere. **Measured:** in a field test that compared two ideation skills, 18 rooms all
|
|
@@ -243,31 +245,32 @@ judge found a real one-cent rounding bug and an overflow.
|
|
|
243
245
|
|
|
244
246
|
### Swap
|
|
245
247
|
|
|
246
|
-
Swap
|
|
247
|
-
|
|
248
|
-
effect (which agent does better), and, where an agent can continue
|
|
249
|
-
|
|
250
|
-
be zero. Swap checks this, and the check is called the
|
|
251
|
-
|
|
248
|
+
Swap stops two runs at several handover points. At each one, each agent continues each run's
|
|
249
|
+
workspace from a handover note. Swap reports a workspace effect (how much the state of the
|
|
250
|
+
workspace matters), an agent effect (which agent does better), and, where an agent can continue
|
|
251
|
+
its own saved session, the note cost. At the handover point 0 both workspaces are the untouched
|
|
252
|
+
project, so the workspace effect there must be zero. Swap checks this, and the check is called the
|
|
253
|
+
start check. When the workspace effect and the agent effect are within 0.1 of each other, Swap
|
|
254
|
+
says they explain the failure about equally.
|
|
252
255
|
|
|
253
|
-
**Measured:** against scripted models with a known truth, the
|
|
254
|
-
passed from the
|
|
255
|
-
to cause.
|
|
256
|
+
**Measured:** against scripted models with a known truth, the start check passed and the blame
|
|
257
|
+
passed from the agent to the workspace between the handovers at 0 and 0.5, as the scripts were
|
|
258
|
+
written to cause.
|
|
256
259
|
|
|
257
260
|
### Cliff
|
|
258
261
|
|
|
259
262
|
Cliff restarts a failed run from its capsules and names the step after which it no longer
|
|
260
|
-
succeeds.
|
|
261
|
-
|
|
262
|
-
and the first bad step, then adds
|
|
263
|
-
the budget is spent. It reports
|
|
264
|
-
|
|
263
|
+
succeeds. Fresh starts are the restarts from the beginning. A step counts as good while its
|
|
264
|
+
restarts pass at least half as often as a fresh start. Cliff halves the range between the last
|
|
265
|
+
good and the first bad step, then adds restarts to the two neighbours until their likely ranges
|
|
266
|
+
separate or the budget is spent. It reports that the task is too hard for the model instead of a
|
|
267
|
+
step only when the upper end of the fresh starts' likely range (a 95% interval) lies below 0.2.
|
|
265
268
|
|
|
266
269
|
**Measured:** against a scripted model whose wrong step is known, Cliff found step 4 with
|
|
267
|
-
separated
|
|
270
|
+
separated likely ranges in 18 restarts. Cliff stays experimental, because neither of two real test
|
|
268
271
|
rounds had a failure that sat in one step (see [Measurements](#measurements)). Swap already
|
|
269
|
-
shows, at three
|
|
270
|
-
Swap shows that rate dropping between two
|
|
272
|
+
shows, at three handover points, how often an agent passes when it continues its own workspace. When a real
|
|
273
|
+
Swap shows that rate dropping between two handover points, Cliff can narrow the drop down to one step. Only
|
|
271
274
|
then does Cliff get new experiments.
|
|
272
275
|
|
|
273
276
|
### Tokens and cost
|
|
@@ -288,7 +291,9 @@ judge read 186.4k tokens in and $0.093 in total, the Codex part estimated from L
|
|
|
288
291
|
|
|
289
292
|
Calibrate, Swap and Cliff write everything into a run directory: a journal (one JSON line per
|
|
290
293
|
event), a `recipe.json`, a `usage.json` with the totals, and the live output of every agent. The dashboard reads only the run
|
|
291
|
-
directory, so `tare watch` shows a finished run the same way as a live one.
|
|
294
|
+
directory, so `tare watch` shows a finished run the same way as a live one. Each run page opens with
|
|
295
|
+
the answer: a sentence from fixed templates per verdict, worked out from the run directory the way
|
|
296
|
+
Calibrate, Swap and Cliff decide, so a run recorded by an older tare reads like a new one. The recipe holds the
|
|
292
297
|
command, a fingerprint of the project and the agent versions. `tare rerun` repeats a run and
|
|
293
298
|
names what has changed since.
|
|
294
299
|
|
|
@@ -299,19 +304,19 @@ macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
|
299
304
|
|
|
300
305
|
| What | Agent and version | Result | Source |
|
|
301
306
|
|---|---|---|---|
|
|
302
|
-
|
|
|
303
|
-
|
|
|
304
|
-
|
|
|
305
|
-
|
|
|
307
|
+
| Control run and room, fake model server | Claude Code 2.1.289 | the control run showed every kind of context, the room only the email | [results](experiments/dirty-twin/RESULTS.md) |
|
|
308
|
+
| Control run and room | Codex CLI 0.160.0 | the room read `tare: 0.00` | [results](experiments/dirty-twin-codex/RESULTS.md) |
|
|
309
|
+
| Control run and room | Pi 1.0.2 | the room read `tare: 0.00` without flags | [results](experiments/dirty-twin-pi/RESULTS.md) |
|
|
310
|
+
| Control run and room | Antigravity CLI 1.2.16 | the room read `tare: 0.00` without flags, and `~/.gemini` stayed unchanged | [results](experiments/dirty-twin-agy/RESULTS.md) |
|
|
306
311
|
| Rooms given copies of real settings on purpose | all four | Claude Code 4 leaks, Codex 6, Pi 5, Antigravity CLI 3, each named with its source | the results above |
|
|
307
|
-
| Scripted Cliff and Swap | Claude Code against scripted models | Cliff found step 4. Swap passed the
|
|
312
|
+
| Scripted Cliff and Swap | Claude Code against scripted models | Cliff found step 4. Swap passed the start check and found the blame passing from the agent to the workspace between the handovers at 0 and 0.5 | `experiments/cliff-scripted`, `experiments/swap-scripted` |
|
|
308
313
|
| Real task 1: a data migration | Claude Code with Sonnet and Haiku, Codex | Deleting the source file after a wrong-encoding import was planned as the point of no return. Swap showed that the workspace could still be repaired. Haiku passed about 1 in 14 runs, Codex with gpt-6.1-sol 4 of 4 | [results](experiments/migration/RESULTS.md) |
|
|
309
314
|
| Real task 2: a web page judged by Sonnet | Claude Code with Haiku, Codex with gpt-6.1-sol | Swap confirmed all four predictions of the plan written before the runs. Cliff called a failed run unlucky, and a later rescoring of the same page confirmed it | [results](experiments/html/RESULTS.md) |
|
|
310
315
|
| Field test: two ideation skills compared | Claude Code 2.1.289 with Opus 5.5 | all 18 rooms read `tare: 0.00`, including a skill that started `claude -p` inside the room | not published |
|
|
311
316
|
| macOS: what a Seatbelt room has to handle | Claude Code 2.1.289, Codex CLI 0.159.0, Pi 0.99.1 | Claude Code's login is in the Keychain. With only the home directory denied, `/var/folders`, `/private/tmp` and the pasteboard stayed readable. Agents' own sandboxes fail inside a room (`sandbox_apply: Operation not permitted`) | comments on epic #85 |
|
|
312
317
|
| macOS: a Seatbelt room with the real login | same | each agent answered one prompt. The room and its login copy were removed afterwards, and the Keychain item was unchanged (digest before and after). `tare probe` read `tare: 0.00` for all three | comments on epic #85 |
|
|
313
|
-
| macOS:
|
|
314
|
-
| macOS: scripted Cliff and Swap, the judge | Claude Code 2.1.290 against scripted models; Microsoft Edge for the judge's screenshot | Cliff found step 4. Swap passed the
|
|
318
|
+
| macOS: control run, Seatbelt room, and parts of the real setup copied into the room | same | every room read `tare: 0.00`. Each copied part that reached the request was named with its source, and nothing under the home directory, under `/var/folders`, in `/private/tmp` or on the pasteboard was reachable from inside. Two kinds of Codex context are invisible to the probe on any system (#109) | [results](experiments/dirty-twin-macos/RESULTS.md) |
|
|
319
|
+
| macOS: scripted Cliff and Swap, the judge | Claude Code 2.1.290 against scripted models; Microsoft Edge for the judge's screenshot | Cliff found step 4. Swap passed the start check and found the blame passing from the agent to the workspace between the handovers at 0 and 0.5. `tare judge` rendered a page and scored it. With a project that turns Claude Code's sandbox on, every Bash call failed in a Seatbelt room until tare turned it off | comments on epic #85 |
|
|
315
320
|
|
|
316
321
|
## Open questions and ideas
|
|
317
322
|
|
|
@@ -321,11 +326,11 @@ macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
|
321
326
|
a fresh room never did. Cached account data is the suspected cause. Not tested.
|
|
322
327
|
- **Claude Code login renewal:** whether a renewal inside the room logs out the real session. Not
|
|
323
328
|
tested, so tare avoids it.
|
|
324
|
-
- **Codex
|
|
329
|
+
- **Codex handover notes:** Codex can run several commands in one tool call, which makes its handover notes
|
|
325
330
|
long.
|
|
326
|
-
- **Cost:** every
|
|
331
|
+
- **Cost:** every restart is a full agent run, so a Cliff search or a Swap costs as much as its restarts.
|
|
327
332
|
- **macOS:** the room is built on Seatbelt, released in 0.3.0 (epic #85). Not done: the
|
|
328
|
-
Antigravity CLI, whose
|
|
333
|
+
Antigravity CLI, whose control run needs Linux's overlay, so tare stops with a message there.
|
|
329
334
|
Not tested: whether a program started through launchd or XPC runs outside the room's profile.
|
|
330
335
|
- **Other probes, not built:** weighing the first request's input tokens before and after adding
|
|
331
336
|
junk to the real setup, and file tripwires that report which secret files a run opened.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tare-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Prove your coding agent starts without your personal setup before you measure it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/a-ratz/tare
|
|
6
6
|
Project-URL: Repository, https://github.com/a-ratz/tare
|
|
@@ -68,7 +68,7 @@ WSL, and with the first three on macOS.
|
|
|
68
68
|
```text
|
|
69
69
|
$ tare probe claude
|
|
70
70
|
tare probe · claude 2.1.289 · /home/me/tare
|
|
71
|
-
control
|
|
71
|
+
control your real setup shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
72
72
|
leaks none
|
|
73
73
|
declared
|
|
74
74
|
email account email subscription login
|
|
@@ -93,13 +93,21 @@ On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). Th
|
|
|
93
93
|
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
94
94
|
every line of the output above.
|
|
95
95
|
|
|
96
|
+
To compare agents, skills or models, `tare calibrate`, `tare swap` and `tare cliff` start many runs
|
|
97
|
+
and open a live dashboard that leads with the answer:
|
|
98
|
+
|
|
99
|
+
<picture>
|
|
100
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-dark.png">
|
|
101
|
+
<img alt="tare's dashboard for a Calibrate run: Claude Code scored higher, 68 against 15 out of 100, with the scores of every run" src="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-light.png">
|
|
102
|
+
</picture>
|
|
103
|
+
|
|
96
104
|
## Features
|
|
97
105
|
|
|
98
106
|
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
99
107
|
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
100
108
|
because an agent asked about its own context can refuse or be wrong.
|
|
101
109
|
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
102
|
-
- **Knows when it cannot see.** A
|
|
110
|
+
- **Knows when it cannot see.** A control run starts the agent in your real setup. If the probe could
|
|
103
111
|
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
104
112
|
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
105
113
|
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
@@ -118,8 +126,8 @@ every line of the output above.
|
|
|
118
126
|
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
119
127
|
from its saved steps and names the step after which it no longer passes.
|
|
120
128
|
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
121
|
-
- **Live dashboard, cost and reruns.**
|
|
122
|
-
and cost, and `tare rerun` repeats a run from its recipe.
|
|
129
|
+
- **Live dashboard, cost and reruns.** The dashboard leads with each run's answer and follows it
|
|
130
|
+
live, each report ends with tokens and cost, and `tare rerun` repeats a run from its recipe.
|
|
123
131
|
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
124
132
|
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
125
133
|
|
|
@@ -43,7 +43,7 @@ WSL, and with the first three on macOS.
|
|
|
43
43
|
```text
|
|
44
44
|
$ tare probe claude
|
|
45
45
|
tare probe · claude 2.1.289 · /home/me/tare
|
|
46
|
-
control
|
|
46
|
+
control your real setup shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
47
47
|
leaks none
|
|
48
48
|
declared
|
|
49
49
|
email account email subscription login
|
|
@@ -68,13 +68,21 @@ On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). Th
|
|
|
68
68
|
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
69
69
|
every line of the output above.
|
|
70
70
|
|
|
71
|
+
To compare agents, skills or models, `tare calibrate`, `tare swap` and `tare cliff` start many runs
|
|
72
|
+
and open a live dashboard that leads with the answer:
|
|
73
|
+
|
|
74
|
+
<picture>
|
|
75
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-dark.png">
|
|
76
|
+
<img alt="tare's dashboard for a Calibrate run: Claude Code scored higher, 68 against 15 out of 100, with the scores of every run" src="https://raw.githubusercontent.com/a-ratz/tare/main/docs/assets/dashboard-calibrate-light.png">
|
|
77
|
+
</picture>
|
|
78
|
+
|
|
71
79
|
## Features
|
|
72
80
|
|
|
73
81
|
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
74
82
|
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
75
83
|
because an agent asked about its own context can refuse or be wrong.
|
|
76
84
|
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
77
|
-
- **Knows when it cannot see.** A
|
|
85
|
+
- **Knows when it cannot see.** A control run starts the agent in your real setup. If the probe could
|
|
78
86
|
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
79
87
|
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
80
88
|
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
@@ -93,8 +101,8 @@ every line of the output above.
|
|
|
93
101
|
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
94
102
|
from its saved steps and names the step after which it no longer passes.
|
|
95
103
|
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
96
|
-
- **Live dashboard, cost and reruns.**
|
|
97
|
-
and cost, and `tare rerun` repeats a run from its recipe.
|
|
104
|
+
- **Live dashboard, cost and reruns.** The dashboard leads with each run's answer and follows it
|
|
105
|
+
live, each report ends with tokens and cost, and `tare rerun` repeats a run from its recipe.
|
|
98
106
|
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
99
107
|
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
100
108
|
|
|
@@ -526,7 +526,7 @@ class Pi:
|
|
|
526
526
|
|
|
527
527
|
@contextmanager
|
|
528
528
|
def twin(self, real: Real, fake_url: str):
|
|
529
|
-
"""The
|
|
529
|
+
"""The control run runs on a copy of the user's agent directory; the real one is never written."""
|
|
530
530
|
with tempfile.TemporaryDirectory(prefix="tare-pi-twin-") as copy:
|
|
531
531
|
agent_dir = Path(copy) / "agent"
|
|
532
532
|
shutil.copytree(real.config, agent_dir, symlinks=True, ignore=shutil.ignore_patterns("sessions"))
|
|
@@ -641,7 +641,7 @@ class Antigravity:
|
|
|
641
641
|
|
|
642
642
|
def discover(self) -> Real:
|
|
643
643
|
if sys.platform == "darwin":
|
|
644
|
-
raise TareError("the Antigravity CLI is not supported on macOS yet: its
|
|
644
|
+
raise TareError("the Antigravity CLI is not supported on macOS yet: its control run needs Linux's overlay "
|
|
645
645
|
"over ~/.gemini (epic #85)")
|
|
646
646
|
home = Path.home()
|
|
647
647
|
return Real(home, home / ".gemini" / "antigravity-cli", _which("agy"))
|
|
@@ -679,7 +679,7 @@ class Antigravity:
|
|
|
679
679
|
|
|
680
680
|
@contextmanager
|
|
681
681
|
def twin(self, real: Real, fake_url: str):
|
|
682
|
-
"""The
|
|
682
|
+
"""The control run sees the real setup, but agy writes into ~/.gemini on every run (its
|
|
683
683
|
conversations, caches, onboarding state). So it runs with an overlay over ~/.gemini
|
|
684
684
|
whose writes go to a tmpfs that dies with the run; see twin_argv."""
|
|
685
685
|
mount = tempfile.mkdtemp(prefix="tare-agy-twin-")
|