tare-cli 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tare_cli-0.2.0 → tare_cli-0.3.1}/CONCEPT.md +66 -15
- tare_cli-0.3.1/PKG-INFO +172 -0
- tare_cli-0.3.1/README.md +147 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/pyproject.toml +2 -1
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/agents.py +146 -44
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/capsule.py +30 -27
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/cli.py +5 -4
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/judge.py +68 -14
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/probe.py +66 -25
- tare_cli-0.3.1/src/tare/room.py +286 -0
- tare_cli-0.3.1/tests/conftest.py +16 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_pi.py +20 -1
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_probe.py +50 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_room.py +48 -0
- tare_cli-0.3.1/tests/test_seatbelt.py +272 -0
- tare_cli-0.2.0/PKG-INFO +0 -505
- tare_cli-0.2.0/README.md +0 -481
- tare_cli-0.2.0/src/tare/room.py +0 -67
- {tare_cli-0.2.0 → tare_cli-0.3.1}/.gitignore +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/LICENSE +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/__init__.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/calibrate.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/cliff.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/dashboard.html +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/dashboard.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/fake.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/journal.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/overview.html +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/recipe.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/swap.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/trail.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/src/tare/usage.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_agents.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_agy.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_calibrate.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_cliff.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_dashboard.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_fake.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_judge.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_swap.py +0 -0
- {tare_cli-0.2.0 → tare_cli-0.3.1}/tests/test_usage.py +0 -0
|
@@ -4,12 +4,13 @@ tare starts a coding agent in an isolated room without your personal setup, and
|
|
|
4
4
|
clean before the run. It prints `tare: 0.00`, names every leak and where it came from, or says
|
|
5
5
|
that it could not see your setup (`tare: not proven (blind)`). It refuses to start the agent
|
|
6
6
|
until the reading is `tare: 0.00`, unless you pass `--allow-dirty`. It supports Claude Code, Codex, Pi
|
|
7
|
-
and the Antigravity CLI (`agy`) on Linux and WSL.
|
|
7
|
+
and the Antigravity CLI (`agy`) on Linux and WSL, and Claude Code, Codex and Pi on macOS.
|
|
8
8
|
|
|
9
|
-
This document describes the design and the measurements behind it. The
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
This document describes the design and the measurements behind it. The documentation explains the
|
|
10
|
+
words this document uses: room, probe, reading, leak, declared and dirty twin in
|
|
11
|
+
[docs/words.md](docs/words.md), check, tail and handoff in [docs/measuring.md](docs/measuring.md).
|
|
12
|
+
Every claim marked **measured** was observed on WSL2. The section [Measurements](#measurements)
|
|
13
|
+
lists the agent versions and links each result.
|
|
13
14
|
|
|
14
15
|
## Why
|
|
15
16
|
|
|
@@ -64,7 +65,9 @@ your instructions, memories, plugins, hooks and skills. That is not enough.
|
|
|
64
65
|
credential files, and called it reconnaissance. That is correct behaviour, but useless for a
|
|
65
66
|
probe. The probe must not depend on the agent's cooperation or honesty.
|
|
66
67
|
- **Reach:** a plain script runs inside the room and looks for your files and inherited
|
|
67
|
-
environment variables.
|
|
68
|
+
environment variables. On macOS it also looks for `~/Library` (Keychains, Preferences,
|
|
69
|
+
Application Support), your directory under `/var/folders`, Claude Code's temporary files in
|
|
70
|
+
`/private/tmp`, and whether the pasteboard can be read.
|
|
68
71
|
- **Context:** tare points the real agent CLI at a fake model server on your machine. The
|
|
69
72
|
server keeps the request, which is the context the agent CLI put together, and answers "ok".
|
|
70
73
|
The idea is the blank run from analytical chemistry, which runs the whole procedure without a
|
|
@@ -83,18 +86,51 @@ your instructions, memories, plugins, hooks and skills. That is not enough.
|
|
|
83
86
|
|
|
84
87
|
### Room
|
|
85
88
|
|
|
86
|
-
A
|
|
89
|
+
A backend builds the room, and every path the agent sees comes from the room it built: its
|
|
90
|
+
home, the project and a run's store. On Linux and WSL the backend is bubblewrap, which mounts
|
|
91
|
+
them at fixed paths. A backend that cannot mount, such as Seatbelt on macOS, uses the host paths
|
|
92
|
+
instead, and the adapters follow (the trust entry, the session directories, the config
|
|
93
|
+
variables).
|
|
94
|
+
|
|
95
|
+
A bubblewrap room is a sandbox for one agent run: an empty `/home` with the room's own home
|
|
87
96
|
directory at `/home/tare`, a fresh copy of the login, the agent's executable, the project at
|
|
88
97
|
`/work`, and the environment allowlist. `tare claude` and the other agent commands mount your
|
|
89
98
|
project directly, so the agent changes your real files. Calibrate, Swap and Cliff give every run
|
|
90
99
|
its own copy of the project. The room has network access and a working login. It keeps your setup out of the measurement. It does not protect your machine from a hostile
|
|
91
100
|
agent.
|
|
92
101
|
|
|
102
|
+
A Seatbelt room (macOS) cannot mount anything. It is one directory under `/private/tmp` that
|
|
103
|
+
holds the room's home, an APFS clone of the project and clones of the agent's executable, placed
|
|
104
|
+
where bubblewrap would mount them (`opt/agent/...`). A clone shares the original's disk blocks,
|
|
105
|
+
so it costs next to nothing. The profile (`sandbox-exec`) denies your home directory, your
|
|
106
|
+
directory under `/var/folders` (other programs' temporary files and caches), the rest of
|
|
107
|
+
`/private/tmp` (where Claude Code keeps its temporary files), `/private/var/tmp`,
|
|
108
|
+
`/Users/Shared` and the pasteboard. Then it allows the room. When the room closes, the run's
|
|
109
|
+
changes to the clone are copied back to the project. A file the run did not touch stays as it
|
|
110
|
+
is. On macOS, Claude Code keeps its login in the Keychain. tare reads it there the way Claude
|
|
111
|
+
Code does and gives the room a copy as a file, which Claude Code uses when the room has no
|
|
112
|
+
Keychain item. No further Seatbelt profile can start inside a Seatbelt room, so the agents' own
|
|
113
|
+
sandboxes do not work there. Codex runs without its sandbox anyway, and tare turns Claude Code's
|
|
114
|
+
off in a macOS room, because a project that turns it on would make every Bash call fail there.
|
|
115
|
+
On macOS, `tar` stores extended attributes as `._` files, so the snapshot hook runs it with
|
|
116
|
+
`COPYFILE_DISABLE=1`. **Measured** on macOS 27.0 (Claude Code 2.1.289, Codex CLI 0.159.0,
|
|
117
|
+
Pi 0.99.1): each agent answered in such a room with its real login. Afterwards the room was
|
|
118
|
+
gone and the Keychain item unchanged, and nothing under the home directory was readable from
|
|
119
|
+
inside (epic #85).
|
|
120
|
+
|
|
93
121
|
Claude Code may replace its login token when it renews it, which could log out your real
|
|
94
122
|
session. This has not been tested, so tare refuses to start when the copy would need renewal
|
|
95
123
|
within the next hour. **Measured:** the Antigravity CLI's Google login keeps its refresh token
|
|
96
124
|
when it renews, so a room may renew its own copy.
|
|
97
125
|
|
|
126
|
+
**Measured** on Ubuntu on WSL2 with Claude Code 2.1.310: Claude Code renews its token only in
|
|
127
|
+
the last minutes before it expires. A session started outside tare with 57 minutes left, and
|
|
128
|
+
another with 7 minutes left, kept the old token; one started with under 6 minutes left renewed
|
|
129
|
+
it, and the new token held for 8 hours. So for most of the refused hour no session outside tare
|
|
130
|
+
can renew the login, and the refusal names the expiry time instead of asking for one. A run
|
|
131
|
+
known to be shorter may ask for less with `TARE_MIN_LOGIN_MINUTES`, never below 10 minutes, so
|
|
132
|
+
the copy does not start inside the renewal window (#126).
|
|
133
|
+
|
|
98
134
|
### Probe and reading
|
|
99
135
|
|
|
100
136
|
The probe has the three parts of layer 3: the agent in the room against the fake model server,
|
|
@@ -121,7 +157,7 @@ lines of your own files that the probe looks for in a request.
|
|
|
121
157
|
| Agent | How the room is set up | Way to the fake model server | Kinds of context the dirty twin showed on the measuring machine |
|
|
122
158
|
|---|---|---|---|
|
|
123
159
|
| Claude Code | `CLAUDE_CONFIG_DIR` and `HOME` in the room, a copy of `.credentials.json`, `--strict-mcp-config --setting-sources project,local` | `ANTHROPIC_BASE_URL` | instructions, home path, MCP servers, skills, plugins, email, reach |
|
|
124
|
-
| Codex | `CODEX_HOME` and `HOME` in the room, a copy of `auth.json` only, `--disable remote_plugin` | `-c openai_base_url=<server>/v1` keeps the ChatGPT login | instructions, memories, home path, skills, reach |
|
|
160
|
+
| Codex | `CODEX_HOME` and `HOME` in the room, a copy of `auth.json` only, `--disable remote_plugin` | `-c openai_base_url=<server>/v1` keeps the ChatGPT login | instructions, memories, home path, skills, plugins, custom agents, reach |
|
|
125
161
|
| Pi | `PI_CODING_AGENT_DIR` and `HOME` in the room, copies of `auth.json` and `models.json`, a minimal `settings.json`, no flags | a provider of its own (`tare`, api `anthropic-messages`), because Pi has no base-URL variable | instruction files above the project, extensions, home path, reach |
|
|
126
162
|
| Antigravity CLI | `HOME` in the room, a copy of `antigravity-oauth-token` and the chosen model, no flags | `CLOUD_CODE_URL` | home path, reach. Global rules (`~/.gemini/GEMINI.md`) and global skills (`~/.gemini/config/skills/`) showed after copies were placed in the setup for the test, because the measuring machine had none. |
|
|
127
163
|
|
|
@@ -132,9 +168,16 @@ What each agent needed:
|
|
|
132
168
|
of its built-in provider that would stop this. So each Codex probe waits about 8 seconds. **Measured:** the captured
|
|
133
169
|
request holds more than `codex debug prompt-input` shows, because Codex's own view leaves out
|
|
134
170
|
the memories. Memories load only when your `config.toml` enables `[features] memories`. Hooks
|
|
135
|
-
did not fire in one-shot runs,
|
|
136
|
-
|
|
137
|
-
|
|
171
|
+
did not fire in one-shot runs, so the probe cannot check hooks for Codex. On the measuring
|
|
172
|
+
machine the first request offered no tools from configured MCP servers. On a Mac (Codex
|
|
173
|
+
0.159.0) it offered one server's tools grouped as `mcp__<server>`, and the probe reads such a
|
|
174
|
+
group as that server's MCP tools (#109). The room has no `config.toml` or `hooks.json` of
|
|
175
|
+
yours, so neither can come in. Codex reports nothing like Claude Code's init event, so the
|
|
176
|
+
probe finds your plugins in the request's skills listing, where each plugin skill is a line
|
|
177
|
+
`- <plugin>:<skill>: …`. Your custom agents (`~/.codex/agents/`, a `.toml` file each) appear
|
|
178
|
+
as roles of the `spawn_agent` tool, `<name>: {` and the description. The probe matches the
|
|
179
|
+
name and the description's first line, so a built-in agent of the same name is not taken for
|
|
180
|
+
yours. A fresh `CODEX_HOME` means
|
|
138
181
|
Codex's default model. Pass `-m` when you compare runs.
|
|
139
182
|
- **Pi:** a fresh agent directory is clean on its own and keeps the project's `AGENTS.md`, so the
|
|
140
183
|
room uses no flags. `--no-context-files` would drop the project's `AGENTS.md`. tare finds Pi's
|
|
@@ -190,7 +233,9 @@ read `tare: 0.00`, including a skill that started `claude -p` inside the room (s
|
|
|
190
233
|
|
|
191
234
|
The judge scores results that no test can decide, such as a web page. tare opens the page in
|
|
192
235
|
headless Google Chrome inside a room with the workspace read-only and no home directory, and
|
|
193
|
-
takes a screenshot.
|
|
236
|
+
takes a screenshot. On macOS it uses Chrome, Chromium or Microsoft Edge from `/Applications`, in
|
|
237
|
+
a Seatbelt room with a browser profile of its own and a mock keychain. Edge keeps running after
|
|
238
|
+
its screenshot, so tare ends it once the file is written. A judge agent (Claude Code with Sonnet by default) then scores the page from
|
|
194
239
|
the screenshot, the source and a rubric, in a fresh room without any agent or model names.
|
|
195
240
|
**Measured:** one test page scored 55, 60, 58, 58, 66 and 62 in six judgements. `tare judge-noise`
|
|
196
241
|
therefore refuses a threshold that lies inside a page's range of scores. On the same page the
|
|
@@ -249,7 +294,8 @@ names what has changed since.
|
|
|
249
294
|
|
|
250
295
|
## Measurements
|
|
251
296
|
|
|
252
|
-
All measurements ran on WSL2 on 2026-10-04 and 2026-10-05.
|
|
297
|
+
All measurements ran on WSL2 on 2026-10-04 and 2026-10-05. The rows that name macOS ran on
|
|
298
|
+
macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
253
299
|
|
|
254
300
|
| What | Agent and version | Result | Source |
|
|
255
301
|
|---|---|---|---|
|
|
@@ -262,6 +308,10 @@ All measurements ran on WSL2 on 2026-10-04 and 2026-10-05.
|
|
|
262
308
|
| Real task 1: a data migration | Claude Code with Sonnet and Haiku, Codex | Deleting the source file after a wrong-encoding import was planned as the point of no return. Swap showed that the workspace could still be repaired. Haiku passed about 1 in 14 runs, Codex with gpt-6.1-sol 4 of 4 | [results](experiments/migration/RESULTS.md) |
|
|
263
309
|
| Real task 2: a web page judged by Sonnet | Claude Code with Haiku, Codex with gpt-6.1-sol | Swap confirmed all four predictions of the plan written before the runs. Cliff called a failed run unlucky, and a later rescoring of the same page confirmed it | [results](experiments/html/RESULTS.md) |
|
|
264
310
|
| Field test: two ideation skills compared | Claude Code 2.1.289 with Opus 5.5 | all 18 rooms read `tare: 0.00`, including a skill that started `claude -p` inside the room | not published |
|
|
311
|
+
| macOS: what a Seatbelt room has to handle | Claude Code 2.1.289, Codex CLI 0.159.0, Pi 0.99.1 | Claude Code's login is in the Keychain. With only the home directory denied, `/var/folders`, `/private/tmp` and the pasteboard stayed readable. Agents' own sandboxes fail inside a room (`sandbox_apply: Operation not permitted`) | comments on epic #85 |
|
|
312
|
+
| macOS: a Seatbelt room with the real login | same | each agent answered one prompt. The room and its login copy were removed afterwards, and the Keychain item was unchanged (digest before and after). `tare probe` read `tare: 0.00` for all three | comments on epic #85 |
|
|
313
|
+
| macOS: dirty twin, Seatbelt room, and parts of the real setup copied into the room | same | every room read `tare: 0.00`. Each copied part that reached the request was named with its source, and nothing under the home directory, under `/var/folders`, in `/private/tmp` or on the pasteboard was reachable from inside. Two kinds of Codex context are invisible to the probe on any system (#109) | [results](experiments/dirty-twin-macos/RESULTS.md) |
|
|
314
|
+
| macOS: scripted Cliff and Swap, the judge | Claude Code 2.1.290 against scripted models; Microsoft Edge for the judge's screenshot | Cliff found step 4. Swap passed the null check and found the blame passing from the model to the workspace between cut 0 and cut 0.5. `tare judge` rendered a page and scored it. With a project that turns Claude Code's sandbox on, every Bash call failed in a Seatbelt room until tare turned it off | comments on epic #85 |
|
|
265
315
|
|
|
266
316
|
## Open questions and ideas
|
|
267
317
|
|
|
@@ -274,8 +324,9 @@ All measurements ran on WSL2 on 2026-10-04 and 2026-10-05.
|
|
|
274
324
|
- **Codex handoffs:** Codex can run several commands in one tool call, which makes its handoffs
|
|
275
325
|
long.
|
|
276
326
|
- **Cost:** every tail is a full agent run, so a Cliff search or a Swap costs as much as its tails.
|
|
277
|
-
- **macOS:**
|
|
278
|
-
|
|
327
|
+
- **macOS:** the room is built on Seatbelt, released in 0.3.0 (epic #85). Not done: the
|
|
328
|
+
Antigravity CLI, whose dirty twin needs Linux's overlay, so tare stops with a message there.
|
|
329
|
+
Not tested: whether a program started through launchd or XPC runs outside the room's profile.
|
|
279
330
|
- **Other probes, not built:** weighing the first request's input tokens before and after adding
|
|
280
331
|
junk to the real setup, and file tripwires that report which secret files a run opened.
|
|
281
332
|
- **Docker:** could contain files where bubblewrap is missing, but does nothing against context
|
tare_cli-0.3.1/PKG-INFO
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tare-cli
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Prove your coding agent starts without your personal setup before you measure it.
|
|
5
|
+
Project-URL: Homepage, https://github.com/a-ratz/tare
|
|
6
|
+
Project-URL: Repository, https://github.com/a-ratz/tare
|
|
7
|
+
Project-URL: Issues, https://github.com/a-ratz/tare/issues
|
|
8
|
+
Project-URL: Concept, https://github.com/a-ratz/tare/blob/main/CONCEPT.md
|
|
9
|
+
Author: Andre Ratzenberger
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agent-evaluation,ai-agents,claude-code,codex,coding-agents,evals,llm-evaluation,sandbox
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
<div align="center">
|
|
27
|
+
<picture>
|
|
28
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-dark.svg">
|
|
29
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg">
|
|
30
|
+
<img alt="tare" src="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg" width="260">
|
|
31
|
+
</picture>
|
|
32
|
+
|
|
33
|
+
<p><b>Zero the scale before you weigh.</b><br>
|
|
34
|
+
Prove your coding agent starts without your personal setup, then measure it.</p>
|
|
35
|
+
</div>
|
|
36
|
+
|
|
37
|
+
<div align="center">
|
|
38
|
+
|
|
39
|
+
[![License: MIT][license-shield]][license-url]
|
|
40
|
+
[![PyPI][version-shield]][version-url]
|
|
41
|
+
[![Python 3.11+][python-shield]][python-url]
|
|
42
|
+
[![Linux | WSL | macOS][platform-shield]][platform-url]
|
|
43
|
+
[![Claude Code | Codex | Pi | Antigravity][agent-shield]][agent-url]
|
|
44
|
+
|
|
45
|
+
</div>
|
|
46
|
+
|
|
47
|
+
<div align="center">
|
|
48
|
+
<a href="#quick-start">Quick start</a> ·
|
|
49
|
+
<a href="#features">Features</a> ·
|
|
50
|
+
<a href="https://github.com/a-ratz/tare/blob/main/docs/README.md">Documentation</a> ·
|
|
51
|
+
<a href="https://github.com/a-ratz/tare/blob/main/CONCEPT.md">Concept</a> ·
|
|
52
|
+
<a href="https://github.com/a-ratz/tare/issues/new?template=bug_report.md">Report Bug</a>
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
<br>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
Your coding agent knows you. Every run loads your global instructions, memories, skills, MCP
|
|
60
|
+
servers and connected accounts. At work that is the point. In an evaluation it is noise. Compare
|
|
61
|
+
runs with and without a skill on your own machine, and both runs carry your setup. You measure
|
|
62
|
+
your machine, not the skill.
|
|
63
|
+
|
|
64
|
+
**tare** starts the agent in a room that holds nothing of yours and proves the room is clean
|
|
65
|
+
before every run. It works with Claude Code, Codex, Pi and Google's Antigravity CLI on Linux and
|
|
66
|
+
WSL, and with the first three on macOS.
|
|
67
|
+
|
|
68
|
+
```text
|
|
69
|
+
$ tare probe claude
|
|
70
|
+
tare probe · claude 2.1.289 · /home/me/tare
|
|
71
|
+
control dirty twin shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
72
|
+
leaks none
|
|
73
|
+
declared
|
|
74
|
+
email account email subscription login
|
|
75
|
+
credentials /home/tare/.claude-config/.credentials.json copy of the login, needed by the CLI
|
|
76
|
+
tare: 0.00
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`tare: 0.00` is the zeroed scale. Nothing of your setup reached the model except what the reading
|
|
80
|
+
declares. A reading with leaks names each one and the file it came from, and the agent does not
|
|
81
|
+
start.
|
|
82
|
+
|
|
83
|
+
## Quick start
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv tool install tare-cli # or: pipx install tare-cli
|
|
87
|
+
tare probe claude # prove the room is clean
|
|
88
|
+
tare claude -- -p "fix the tests" # run Claude Code in it
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). The
|
|
92
|
+
[install guide](https://github.com/a-ratz/tare/blob/main/docs/install.md) has the details per
|
|
93
|
+
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
94
|
+
every line of the output above.
|
|
95
|
+
|
|
96
|
+
## Features
|
|
97
|
+
|
|
98
|
+
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
99
|
+
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
100
|
+
because an agent asked about its own context can refuse or be wrong.
|
|
101
|
+
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
102
|
+
- **Knows when it cannot see.** A dirty twin runs the agent in your real setup. If the probe could
|
|
103
|
+
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
104
|
+
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
105
|
+
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
106
|
+
credentials of your other tools and no secret environment variables. It gets a fresh copy of
|
|
107
|
+
your login, which tare deletes after the run.
|
|
108
|
+
[Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md)
|
|
109
|
+
- **Compares skills, plugins and models.** `tare calibrate` starts each side several times from
|
|
110
|
+
scratch and reports pass rates with their intervals.
|
|
111
|
+
[Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md)
|
|
112
|
+
- **Scores what no test can decide.** `tare judge` lets a second agent score a page by your
|
|
113
|
+
rubric, without agent or model names, and `tare judge-noise` shows how far its scores vary.
|
|
114
|
+
[Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md)
|
|
115
|
+
- **Tells the workspace from the agent.** `tare swap` lets two agents continue each other's
|
|
116
|
+
unfinished work and reports whether the workspace or the agent decided the outcome.
|
|
117
|
+
[Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md)
|
|
118
|
+
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
119
|
+
from its saved steps and names the step after which it no longer passes.
|
|
120
|
+
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
121
|
+
- **Live dashboard, cost and reruns.** A dashboard follows every run, each report ends with tokens
|
|
122
|
+
and cost, and `tare rerun` repeats a run from its recipe.
|
|
123
|
+
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
124
|
+
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
125
|
+
|
|
126
|
+
## When to use
|
|
127
|
+
|
|
128
|
+
| Use tare when | Look elsewhere when |
|
|
129
|
+
|---|---|
|
|
130
|
+
| you compare agent runs with and without a skill, plugin or prompt | you need protection from a hostile agent, because the room has network access and a working login |
|
|
131
|
+
| you benchmark agents on your own machine | you use the Antigravity CLI on macOS (not supported there yet) |
|
|
132
|
+
| you want a fresh-machine run without a fresh machine | you log in with an API key only (untested) |
|
|
133
|
+
| an agent failed a task and you want to know which step lost it | you want a single run explained without running the agent again, because Cliff runs it many times |
|
|
134
|
+
|
|
135
|
+
## Documentation
|
|
136
|
+
|
|
137
|
+
| | |
|
|
138
|
+
|---|---|
|
|
139
|
+
| **Getting started** | [Install](https://github.com/a-ratz/tare/blob/main/docs/install.md) · [The words tare uses](https://github.com/a-ratz/tare/blob/main/docs/words.md) · [Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md) · [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) |
|
|
140
|
+
| **Measuring** | [Overview](https://github.com/a-ratz/tare/blob/main/docs/measuring.md) · [Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md) · [Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md) · [Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md) · [Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md) · [Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md) · [Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) |
|
|
141
|
+
| **Background** | [How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md) · [Concept: the design and every measurement](https://github.com/a-ratz/tare/blob/main/CONCEPT.md) · [Experiments](https://github.com/a-ratz/tare/tree/main/experiments) |
|
|
142
|
+
|
|
143
|
+
## Status
|
|
144
|
+
|
|
145
|
+
tare is on PyPI as `tare-cli`. Claude Code, Codex and Pi run on Linux, WSL and macOS; the
|
|
146
|
+
Antigravity CLI runs on Linux and WSL, and stops with a message on macOS for now. What comes next
|
|
147
|
+
is in the [open epics](https://github.com/a-ratz/tare/issues?q=is%3Aissue+is%3Aopen+label%3Atype%3Aepic).
|
|
148
|
+
|
|
149
|
+
## Contributing
|
|
150
|
+
|
|
151
|
+
Every change starts as an issue. Larger work is an epic, split into features, and one epic
|
|
152
|
+
becomes one branch and one pull request. [AGENTS.md](https://github.com/a-ratz/tare/blob/main/AGENTS.md)
|
|
153
|
+
describes the process, the checks and the conventions. Run the tests with `uv run pytest`.
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT. See [LICENSE](https://github.com/a-ratz/tare/blob/main/LICENSE).
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
Crafted with [Readme Craft](https://github.com/motiful/readme-craft)
|
|
162
|
+
|
|
163
|
+
[license-shield]: https://img.shields.io/badge/license-MIT-blue.svg
|
|
164
|
+
[license-url]: https://github.com/a-ratz/tare/blob/main/LICENSE
|
|
165
|
+
[version-shield]: https://img.shields.io/pypi/v/tare-cli.svg
|
|
166
|
+
[version-url]: https://pypi.org/project/tare-cli/
|
|
167
|
+
[python-shield]: https://img.shields.io/badge/python-3.11%2B-3776AB.svg
|
|
168
|
+
[python-url]: https://www.python.org/
|
|
169
|
+
[platform-shield]: https://img.shields.io/badge/platform-Linux%20%7C%20WSL%20%7C%20macOS-555555.svg
|
|
170
|
+
[platform-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
|
171
|
+
[agent-shield]: https://img.shields.io/badge/agents-Claude%20Code%20%7C%20Codex%20%7C%20Pi%20%7C%20Antigravity-D97757.svg
|
|
172
|
+
[agent-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
tare_cli-0.3.1/README.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-dark.svg">
|
|
4
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg">
|
|
5
|
+
<img alt="tare" src="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg" width="260">
|
|
6
|
+
</picture>
|
|
7
|
+
|
|
8
|
+
<p><b>Zero the scale before you weigh.</b><br>
|
|
9
|
+
Prove your coding agent starts without your personal setup, then measure it.</p>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<div align="center">
|
|
13
|
+
|
|
14
|
+
[![License: MIT][license-shield]][license-url]
|
|
15
|
+
[![PyPI][version-shield]][version-url]
|
|
16
|
+
[![Python 3.11+][python-shield]][python-url]
|
|
17
|
+
[![Linux | WSL | macOS][platform-shield]][platform-url]
|
|
18
|
+
[![Claude Code | Codex | Pi | Antigravity][agent-shield]][agent-url]
|
|
19
|
+
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
<div align="center">
|
|
23
|
+
<a href="#quick-start">Quick start</a> ·
|
|
24
|
+
<a href="#features">Features</a> ·
|
|
25
|
+
<a href="https://github.com/a-ratz/tare/blob/main/docs/README.md">Documentation</a> ·
|
|
26
|
+
<a href="https://github.com/a-ratz/tare/blob/main/CONCEPT.md">Concept</a> ·
|
|
27
|
+
<a href="https://github.com/a-ratz/tare/issues/new?template=bug_report.md">Report Bug</a>
|
|
28
|
+
</div>
|
|
29
|
+
|
|
30
|
+
<br>
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
Your coding agent knows you. Every run loads your global instructions, memories, skills, MCP
|
|
35
|
+
servers and connected accounts. At work that is the point. In an evaluation it is noise. Compare
|
|
36
|
+
runs with and without a skill on your own machine, and both runs carry your setup. You measure
|
|
37
|
+
your machine, not the skill.
|
|
38
|
+
|
|
39
|
+
**tare** starts the agent in a room that holds nothing of yours and proves the room is clean
|
|
40
|
+
before every run. It works with Claude Code, Codex, Pi and Google's Antigravity CLI on Linux and
|
|
41
|
+
WSL, and with the first three on macOS.
|
|
42
|
+
|
|
43
|
+
```text
|
|
44
|
+
$ tare probe claude
|
|
45
|
+
tare probe · claude 2.1.289 · /home/me/tare
|
|
46
|
+
control dirty twin shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
47
|
+
leaks none
|
|
48
|
+
declared
|
|
49
|
+
email account email subscription login
|
|
50
|
+
credentials /home/tare/.claude-config/.credentials.json copy of the login, needed by the CLI
|
|
51
|
+
tare: 0.00
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`tare: 0.00` is the zeroed scale. Nothing of your setup reached the model except what the reading
|
|
55
|
+
declares. A reading with leaks names each one and the file it came from, and the agent does not
|
|
56
|
+
start.
|
|
57
|
+
|
|
58
|
+
## Quick start
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
uv tool install tare-cli # or: pipx install tare-cli
|
|
62
|
+
tare probe claude # prove the room is clean
|
|
63
|
+
tare claude -- -p "fix the tests" # run Claude Code in it
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). The
|
|
67
|
+
[install guide](https://github.com/a-ratz/tare/blob/main/docs/install.md) has the details per
|
|
68
|
+
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
69
|
+
every line of the output above.
|
|
70
|
+
|
|
71
|
+
## Features
|
|
72
|
+
|
|
73
|
+
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
74
|
+
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
75
|
+
because an agent asked about its own context can refuse or be wrong.
|
|
76
|
+
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
77
|
+
- **Knows when it cannot see.** A dirty twin runs the agent in your real setup. If the probe could
|
|
78
|
+
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
79
|
+
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
80
|
+
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
81
|
+
credentials of your other tools and no secret environment variables. It gets a fresh copy of
|
|
82
|
+
your login, which tare deletes after the run.
|
|
83
|
+
[Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md)
|
|
84
|
+
- **Compares skills, plugins and models.** `tare calibrate` starts each side several times from
|
|
85
|
+
scratch and reports pass rates with their intervals.
|
|
86
|
+
[Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md)
|
|
87
|
+
- **Scores what no test can decide.** `tare judge` lets a second agent score a page by your
|
|
88
|
+
rubric, without agent or model names, and `tare judge-noise` shows how far its scores vary.
|
|
89
|
+
[Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md)
|
|
90
|
+
- **Tells the workspace from the agent.** `tare swap` lets two agents continue each other's
|
|
91
|
+
unfinished work and reports whether the workspace or the agent decided the outcome.
|
|
92
|
+
[Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md)
|
|
93
|
+
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
94
|
+
from its saved steps and names the step after which it no longer passes.
|
|
95
|
+
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
96
|
+
- **Live dashboard, cost and reruns.** A dashboard follows every run, each report ends with tokens
|
|
97
|
+
and cost, and `tare rerun` repeats a run from its recipe.
|
|
98
|
+
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
99
|
+
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
100
|
+
|
|
101
|
+
## When to use
|
|
102
|
+
|
|
103
|
+
| Use tare when | Look elsewhere when |
|
|
104
|
+
|---|---|
|
|
105
|
+
| you compare agent runs with and without a skill, plugin or prompt | you need protection from a hostile agent, because the room has network access and a working login |
|
|
106
|
+
| you benchmark agents on your own machine | you use the Antigravity CLI on macOS (not supported there yet) |
|
|
107
|
+
| you want a fresh-machine run without a fresh machine | you log in with an API key only (untested) |
|
|
108
|
+
| an agent failed a task and you want to know which step lost it | you want a single run explained without running the agent again, because Cliff runs it many times |
|
|
109
|
+
|
|
110
|
+
## Documentation
|
|
111
|
+
|
|
112
|
+
| | |
|
|
113
|
+
|---|---|
|
|
114
|
+
| **Getting started** | [Install](https://github.com/a-ratz/tare/blob/main/docs/install.md) · [The words tare uses](https://github.com/a-ratz/tare/blob/main/docs/words.md) · [Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md) · [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) |
|
|
115
|
+
| **Measuring** | [Overview](https://github.com/a-ratz/tare/blob/main/docs/measuring.md) · [Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md) · [Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md) · [Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md) · [Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md) · [Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md) · [Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) |
|
|
116
|
+
| **Background** | [How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md) · [Concept: the design and every measurement](https://github.com/a-ratz/tare/blob/main/CONCEPT.md) · [Experiments](https://github.com/a-ratz/tare/tree/main/experiments) |
|
|
117
|
+
|
|
118
|
+
## Status
|
|
119
|
+
|
|
120
|
+
tare is on PyPI as `tare-cli`. Claude Code, Codex and Pi run on Linux, WSL and macOS; the
|
|
121
|
+
Antigravity CLI runs on Linux and WSL, and stops with a message on macOS for now. What comes next
|
|
122
|
+
is in the [open epics](https://github.com/a-ratz/tare/issues?q=is%3Aissue+is%3Aopen+label%3Atype%3Aepic).
|
|
123
|
+
|
|
124
|
+
## Contributing
|
|
125
|
+
|
|
126
|
+
Every change starts as an issue. Larger work is an epic, split into features, and one epic
|
|
127
|
+
becomes one branch and one pull request. [AGENTS.md](https://github.com/a-ratz/tare/blob/main/AGENTS.md)
|
|
128
|
+
describes the process, the checks and the conventions. Run the tests with `uv run pytest`.
|
|
129
|
+
|
|
130
|
+
## License
|
|
131
|
+
|
|
132
|
+
MIT. See [LICENSE](https://github.com/a-ratz/tare/blob/main/LICENSE).
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
Crafted with [Readme Craft](https://github.com/motiful/readme-craft)
|
|
137
|
+
|
|
138
|
+
[license-shield]: https://img.shields.io/badge/license-MIT-blue.svg
|
|
139
|
+
[license-url]: https://github.com/a-ratz/tare/blob/main/LICENSE
|
|
140
|
+
[version-shield]: https://img.shields.io/pypi/v/tare-cli.svg
|
|
141
|
+
[version-url]: https://pypi.org/project/tare-cli/
|
|
142
|
+
[python-shield]: https://img.shields.io/badge/python-3.11%2B-3776AB.svg
|
|
143
|
+
[python-url]: https://www.python.org/
|
|
144
|
+
[platform-shield]: https://img.shields.io/badge/platform-Linux%20%7C%20WSL%20%7C%20macOS-555555.svg
|
|
145
|
+
[platform-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
|
146
|
+
[agent-shield]: https://img.shields.io/badge/agents-Claude%20Code%20%7C%20Codex%20%7C%20Pi%20%7C%20Antigravity-D97757.svg
|
|
147
|
+
[agent-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tare-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.1"
|
|
4
4
|
description = "Prove your coding agent starts without your personal setup before you measure it."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -13,6 +13,7 @@ classifiers = [
|
|
|
13
13
|
"Development Status :: 3 - Alpha",
|
|
14
14
|
"Environment :: Console",
|
|
15
15
|
"Intended Audience :: Developers",
|
|
16
|
+
"Operating System :: MacOS",
|
|
16
17
|
"Operating System :: POSIX :: Linux",
|
|
17
18
|
"Programming Language :: Python :: 3",
|
|
18
19
|
"Programming Language :: Python :: 3.11",
|