tare-cli 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tare_cli-0.3.0 → tare_cli-0.3.1}/CONCEPT.md +13 -4
- tare_cli-0.3.1/PKG-INFO +172 -0
- tare_cli-0.3.1/README.md +147 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/pyproject.toml +1 -1
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/room.py +23 -4
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_room.py +29 -0
- tare_cli-0.3.0/PKG-INFO +0 -506
- tare_cli-0.3.0/README.md +0 -481
- {tare_cli-0.3.0 → tare_cli-0.3.1}/.gitignore +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/LICENSE +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/__init__.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/agents.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/calibrate.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/capsule.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/cli.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/cliff.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/dashboard.html +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/dashboard.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/fake.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/journal.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/judge.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/overview.html +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/probe.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/recipe.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/swap.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/trail.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/src/tare/usage.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/conftest.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_agents.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_agy.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_calibrate.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_cliff.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_dashboard.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_fake.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_judge.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_pi.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_probe.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_seatbelt.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_swap.py +0 -0
- {tare_cli-0.3.0 → tare_cli-0.3.1}/tests/test_usage.py +0 -0
|
@@ -6,10 +6,11 @@ that it could not see your setup (`tare: not proven (blind)`). It refuses to sta
|
|
|
6
6
|
until the reading is `tare: 0.00`, unless you pass `--allow-dirty`. It supports Claude Code, Codex, Pi
|
|
7
7
|
and the Antigravity CLI (`agy`) on Linux and WSL, and Claude Code, Codex and Pi on macOS.
|
|
8
8
|
|
|
9
|
-
This document describes the design and the measurements behind it. The
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
This document describes the design and the measurements behind it. The documentation explains the
|
|
10
|
+
words this document uses: room, probe, reading, leak, declared and dirty twin in
|
|
11
|
+
[docs/words.md](docs/words.md), check, tail and handoff in [docs/measuring.md](docs/measuring.md).
|
|
12
|
+
Every claim marked **measured** was observed on WSL2. The section [Measurements](#measurements)
|
|
13
|
+
lists the agent versions and links each result.
|
|
13
14
|
|
|
14
15
|
## Why
|
|
15
16
|
|
|
@@ -122,6 +123,14 @@ session. This has not been tested, so tare refuses to start when the copy would
|
|
|
122
123
|
within the next hour. **Measured:** the Antigravity CLI's Google login keeps its refresh token
|
|
123
124
|
when it renews, so a room may renew its own copy.
|
|
124
125
|
|
|
126
|
+
**Measured** on Ubuntu on WSL2 with Claude Code 2.1.310: Claude Code renews its token only in
|
|
127
|
+
the last minutes before it expires. A session started outside tare with 57 minutes left, and
|
|
128
|
+
another with 7 minutes left, kept the old token; one started with under 6 minutes left renewed
|
|
129
|
+
it, and the new token held for 8 hours. So for most of the refused hour no session outside tare
|
|
130
|
+
can renew the login, and the refusal names the expiry time instead of asking for one. A run
|
|
131
|
+
known to be shorter may ask for less with `TARE_MIN_LOGIN_MINUTES`, never below 10 minutes, so
|
|
132
|
+
the copy does not start inside the renewal window (#126).
|
|
133
|
+
|
|
125
134
|
### Probe and reading
|
|
126
135
|
|
|
127
136
|
The probe has the three parts of layer 3: the agent in the room against the fake model server,
|
tare_cli-0.3.1/PKG-INFO
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tare-cli
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Prove your coding agent starts without your personal setup before you measure it.
|
|
5
|
+
Project-URL: Homepage, https://github.com/a-ratz/tare
|
|
6
|
+
Project-URL: Repository, https://github.com/a-ratz/tare
|
|
7
|
+
Project-URL: Issues, https://github.com/a-ratz/tare/issues
|
|
8
|
+
Project-URL: Concept, https://github.com/a-ratz/tare/blob/main/CONCEPT.md
|
|
9
|
+
Author: Andre Ratzenberger
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agent-evaluation,ai-agents,claude-code,codex,coding-agents,evals,llm-evaluation,sandbox
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
<div align="center">
|
|
27
|
+
<picture>
|
|
28
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-dark.svg">
|
|
29
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg">
|
|
30
|
+
<img alt="tare" src="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg" width="260">
|
|
31
|
+
</picture>
|
|
32
|
+
|
|
33
|
+
<p><b>Zero the scale before you weigh.</b><br>
|
|
34
|
+
Prove your coding agent starts without your personal setup, then measure it.</p>
|
|
35
|
+
</div>
|
|
36
|
+
|
|
37
|
+
<div align="center">
|
|
38
|
+
|
|
39
|
+
[![License: MIT][license-shield]][license-url]
|
|
40
|
+
[![PyPI][version-shield]][version-url]
|
|
41
|
+
[![Python 3.11+][python-shield]][python-url]
|
|
42
|
+
[![Linux | WSL | macOS][platform-shield]][platform-url]
|
|
43
|
+
[![Claude Code | Codex | Pi | Antigravity][agent-shield]][agent-url]
|
|
44
|
+
|
|
45
|
+
</div>
|
|
46
|
+
|
|
47
|
+
<div align="center">
|
|
48
|
+
<a href="#quick-start">Quick start</a> ·
|
|
49
|
+
<a href="#features">Features</a> ·
|
|
50
|
+
<a href="https://github.com/a-ratz/tare/blob/main/docs/README.md">Documentation</a> ·
|
|
51
|
+
<a href="https://github.com/a-ratz/tare/blob/main/CONCEPT.md">Concept</a> ·
|
|
52
|
+
<a href="https://github.com/a-ratz/tare/issues/new?template=bug_report.md">Report Bug</a>
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
<br>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
Your coding agent knows you. Every run loads your global instructions, memories, skills, MCP
|
|
60
|
+
servers and connected accounts. At work that is the point. In an evaluation it is noise. Compare
|
|
61
|
+
runs with and without a skill on your own machine, and both runs carry your setup. You measure
|
|
62
|
+
your machine, not the skill.
|
|
63
|
+
|
|
64
|
+
**tare** starts the agent in a room that holds nothing of yours and proves the room is clean
|
|
65
|
+
before every run. It works with Claude Code, Codex, Pi and Google's Antigravity CLI on Linux and
|
|
66
|
+
WSL, and with the first three on macOS.
|
|
67
|
+
|
|
68
|
+
```text
|
|
69
|
+
$ tare probe claude
|
|
70
|
+
tare probe · claude 2.1.289 · /home/me/tare
|
|
71
|
+
control dirty twin shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
72
|
+
leaks none
|
|
73
|
+
declared
|
|
74
|
+
email account email subscription login
|
|
75
|
+
credentials /home/tare/.claude-config/.credentials.json copy of the login, needed by the CLI
|
|
76
|
+
tare: 0.00
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`tare: 0.00` is the zeroed scale. Nothing of your setup reached the model except what the reading
|
|
80
|
+
declares. A reading with leaks names each one and the file it came from, and the agent does not
|
|
81
|
+
start.
|
|
82
|
+
|
|
83
|
+
## Quick start
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv tool install tare-cli # or: pipx install tare-cli
|
|
87
|
+
tare probe claude # prove the room is clean
|
|
88
|
+
tare claude -- -p "fix the tests" # run Claude Code in it
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). The
|
|
92
|
+
[install guide](https://github.com/a-ratz/tare/blob/main/docs/install.md) has the details per
|
|
93
|
+
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
94
|
+
every line of the output above.
|
|
95
|
+
|
|
96
|
+
## Features
|
|
97
|
+
|
|
98
|
+
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
99
|
+
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
100
|
+
because an agent asked about its own context can refuse or be wrong.
|
|
101
|
+
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
102
|
+
- **Knows when it cannot see.** A dirty twin runs the agent in your real setup. If the probe could
|
|
103
|
+
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
104
|
+
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
105
|
+
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
106
|
+
credentials of your other tools and no secret environment variables. It gets a fresh copy of
|
|
107
|
+
your login, which tare deletes after the run.
|
|
108
|
+
[Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md)
|
|
109
|
+
- **Compares skills, plugins and models.** `tare calibrate` starts each side several times from
|
|
110
|
+
scratch and reports pass rates with their intervals.
|
|
111
|
+
[Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md)
|
|
112
|
+
- **Scores what no test can decide.** `tare judge` lets a second agent score a page by your
|
|
113
|
+
rubric, without agent or model names, and `tare judge-noise` shows how far its scores vary.
|
|
114
|
+
[Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md)
|
|
115
|
+
- **Tells the workspace from the agent.** `tare swap` lets two agents continue each other's
|
|
116
|
+
unfinished work and reports whether the workspace or the agent decided the outcome.
|
|
117
|
+
[Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md)
|
|
118
|
+
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
119
|
+
from its saved steps and names the step after which it no longer passes.
|
|
120
|
+
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
121
|
+
- **Live dashboard, cost and reruns.** A dashboard follows every run, each report ends with tokens
|
|
122
|
+
and cost, and `tare rerun` repeats a run from its recipe.
|
|
123
|
+
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
124
|
+
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
125
|
+
|
|
126
|
+
## When to use
|
|
127
|
+
|
|
128
|
+
| Use tare when | Look elsewhere when |
|
|
129
|
+
|---|---|
|
|
130
|
+
| you compare agent runs with and without a skill, plugin or prompt | you need protection from a hostile agent, because the room has network access and a working login |
|
|
131
|
+
| you benchmark agents on your own machine | you use the Antigravity CLI on macOS (not supported there yet) |
|
|
132
|
+
| you want a fresh-machine run without a fresh machine | you log in with an API key only (untested) |
|
|
133
|
+
| an agent failed a task and you want to know which step lost it | you want a single run explained without running the agent again, because Cliff runs it many times |
|
|
134
|
+
|
|
135
|
+
## Documentation
|
|
136
|
+
|
|
137
|
+
| | |
|
|
138
|
+
|---|---|
|
|
139
|
+
| **Getting started** | [Install](https://github.com/a-ratz/tare/blob/main/docs/install.md) · [The words tare uses](https://github.com/a-ratz/tare/blob/main/docs/words.md) · [Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md) · [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) |
|
|
140
|
+
| **Measuring** | [Overview](https://github.com/a-ratz/tare/blob/main/docs/measuring.md) · [Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md) · [Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md) · [Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md) · [Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md) · [Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md) · [Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) |
|
|
141
|
+
| **Background** | [How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md) · [Concept: the design and every measurement](https://github.com/a-ratz/tare/blob/main/CONCEPT.md) · [Experiments](https://github.com/a-ratz/tare/tree/main/experiments) |
|
|
142
|
+
|
|
143
|
+
## Status
|
|
144
|
+
|
|
145
|
+
tare is on PyPI as `tare-cli`. Claude Code, Codex and Pi run on Linux, WSL and macOS; the
|
|
146
|
+
Antigravity CLI runs on Linux and WSL, and stops with a message on macOS for now. What comes next
|
|
147
|
+
is in the [open epics](https://github.com/a-ratz/tare/issues?q=is%3Aissue+is%3Aopen+label%3Atype%3Aepic).
|
|
148
|
+
|
|
149
|
+
## Contributing
|
|
150
|
+
|
|
151
|
+
Every change starts as an issue. Larger work is an epic, split into features, and one epic
|
|
152
|
+
becomes one branch and one pull request. [AGENTS.md](https://github.com/a-ratz/tare/blob/main/AGENTS.md)
|
|
153
|
+
describes the process, the checks and the conventions. Run the tests with `uv run pytest`.
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT. See [LICENSE](https://github.com/a-ratz/tare/blob/main/LICENSE).
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
Crafted with [Readme Craft](https://github.com/motiful/readme-craft)
|
|
162
|
+
|
|
163
|
+
[license-shield]: https://img.shields.io/badge/license-MIT-blue.svg
|
|
164
|
+
[license-url]: https://github.com/a-ratz/tare/blob/main/LICENSE
|
|
165
|
+
[version-shield]: https://img.shields.io/pypi/v/tare-cli.svg
|
|
166
|
+
[version-url]: https://pypi.org/project/tare-cli/
|
|
167
|
+
[python-shield]: https://img.shields.io/badge/python-3.11%2B-3776AB.svg
|
|
168
|
+
[python-url]: https://www.python.org/
|
|
169
|
+
[platform-shield]: https://img.shields.io/badge/platform-Linux%20%7C%20WSL%20%7C%20macOS-555555.svg
|
|
170
|
+
[platform-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
|
171
|
+
[agent-shield]: https://img.shields.io/badge/agents-Claude%20Code%20%7C%20Codex%20%7C%20Pi%20%7C%20Antigravity-D97757.svg
|
|
172
|
+
[agent-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
tare_cli-0.3.1/README.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-dark.svg">
|
|
4
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg">
|
|
5
|
+
<img alt="tare" src="https://raw.githubusercontent.com/a-ratz/tare/main/.github/logo-light.svg" width="260">
|
|
6
|
+
</picture>
|
|
7
|
+
|
|
8
|
+
<p><b>Zero the scale before you weigh.</b><br>
|
|
9
|
+
Prove your coding agent starts without your personal setup, then measure it.</p>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<div align="center">
|
|
13
|
+
|
|
14
|
+
[![License: MIT][license-shield]][license-url]
|
|
15
|
+
[![PyPI][version-shield]][version-url]
|
|
16
|
+
[![Python 3.11+][python-shield]][python-url]
|
|
17
|
+
[![Linux | WSL | macOS][platform-shield]][platform-url]
|
|
18
|
+
[![Claude Code | Codex | Pi | Antigravity][agent-shield]][agent-url]
|
|
19
|
+
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
<div align="center">
|
|
23
|
+
<a href="#quick-start">Quick start</a> ·
|
|
24
|
+
<a href="#features">Features</a> ·
|
|
25
|
+
<a href="https://github.com/a-ratz/tare/blob/main/docs/README.md">Documentation</a> ·
|
|
26
|
+
<a href="https://github.com/a-ratz/tare/blob/main/CONCEPT.md">Concept</a> ·
|
|
27
|
+
<a href="https://github.com/a-ratz/tare/issues/new?template=bug_report.md">Report Bug</a>
|
|
28
|
+
</div>
|
|
29
|
+
|
|
30
|
+
<br>
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
Your coding agent knows you. Every run loads your global instructions, memories, skills, MCP
|
|
35
|
+
servers and connected accounts. At work that is the point. In an evaluation it is noise. Compare
|
|
36
|
+
runs with and without a skill on your own machine, and both runs carry your setup. You measure
|
|
37
|
+
your machine, not the skill.
|
|
38
|
+
|
|
39
|
+
**tare** starts the agent in a room that holds nothing of yours and proves the room is clean
|
|
40
|
+
before every run. It works with Claude Code, Codex, Pi and Google's Antigravity CLI on Linux and
|
|
41
|
+
WSL, and with the first three on macOS.
|
|
42
|
+
|
|
43
|
+
```text
|
|
44
|
+
$ tare probe claude
|
|
45
|
+
tare probe · claude 2.1.289 · /home/me/tare
|
|
46
|
+
control dirty twin shows: instructions, home path, mcp, skills, plugins, email, reach
|
|
47
|
+
leaks none
|
|
48
|
+
declared
|
|
49
|
+
email account email subscription login
|
|
50
|
+
credentials /home/tare/.claude-config/.credentials.json copy of the login, needed by the CLI
|
|
51
|
+
tare: 0.00
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`tare: 0.00` is the zeroed scale. Nothing of your setup reached the model except what the reading
|
|
55
|
+
declares. A reading with leaks names each one and the file it came from, and the agent does not
|
|
56
|
+
start.
|
|
57
|
+
|
|
58
|
+
## Quick start
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
uv tool install tare-cli # or: pipx install tare-cli
|
|
62
|
+
tare probe claude # prove the room is clean
|
|
63
|
+
tare claude -- -p "fix the tests" # run Claude Code in it
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
On Linux and WSL, tare also needs bubblewrap (`sudo apt install bubblewrap`). The
|
|
67
|
+
[install guide](https://github.com/a-ratz/tare/blob/main/docs/install.md) has the details per
|
|
68
|
+
platform, and [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) explains
|
|
69
|
+
every line of the output above.
|
|
70
|
+
|
|
71
|
+
## Features
|
|
72
|
+
|
|
73
|
+
- **Measures what the model receives.** The probe records the request the agent sends to a fake
|
|
74
|
+
model server on your machine. It makes no paid calls and never asks the agent about itself,
|
|
75
|
+
because an agent asked about its own context can refuse or be wrong.
|
|
76
|
+
[How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md)
|
|
77
|
+
- **Knows when it cannot see.** A dirty twin runs the agent in your real setup. If the probe could
|
|
78
|
+
not see your setup there, the reading is `tare: not proven (blind)` instead of a `0.00` that
|
|
79
|
+
proves nothing. [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md)
|
|
80
|
+
- **Keeps your files out and your login working.** The room has no home directory of yours, no
|
|
81
|
+
credentials of your other tools and no secret environment variables. It gets a fresh copy of
|
|
82
|
+
your login, which tare deletes after the run.
|
|
83
|
+
[Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md)
|
|
84
|
+
- **Compares skills, plugins and models.** `tare calibrate` starts each side several times from
|
|
85
|
+
scratch and reports pass rates with their intervals.
|
|
86
|
+
[Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md)
|
|
87
|
+
- **Scores what no test can decide.** `tare judge` lets a second agent score a page by your
|
|
88
|
+
rubric, without agent or model names, and `tare judge-noise` shows how far its scores vary.
|
|
89
|
+
[Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md)
|
|
90
|
+
- **Tells the workspace from the agent.** `tare swap` lets two agents continue each other's
|
|
91
|
+
unfinished work and reports whether the workspace or the agent decided the outcome.
|
|
92
|
+
[Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md)
|
|
93
|
+
- **Finds the step where a failed run was lost** (experimental). `tare cliff` restarts the run
|
|
94
|
+
from its saved steps and names the step after which it no longer passes.
|
|
95
|
+
[Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md)
|
|
96
|
+
- **Live dashboard, cost and reruns.** A dashboard follows every run, each report ends with tokens
|
|
97
|
+
and cost, and `tare rerun` repeats a run from its recipe.
|
|
98
|
+
[Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) ·
|
|
99
|
+
[Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md)
|
|
100
|
+
|
|
101
|
+
## When to use
|
|
102
|
+
|
|
103
|
+
| Use tare when | Look elsewhere when |
|
|
104
|
+
|---|---|
|
|
105
|
+
| you compare agent runs with and without a skill, plugin or prompt | you need protection from a hostile agent, because the room has network access and a working login |
|
|
106
|
+
| you benchmark agents on your own machine | you use the Antigravity CLI on macOS (not supported there yet) |
|
|
107
|
+
| you want a fresh-machine run without a fresh machine | you log in with an API key only (untested) |
|
|
108
|
+
| an agent failed a task and you want to know which step lost it | you want a single run explained without running the agent again, because Cliff runs it many times |
|
|
109
|
+
|
|
110
|
+
## Documentation
|
|
111
|
+
|
|
112
|
+
| | |
|
|
113
|
+
|---|---|
|
|
114
|
+
| **Getting started** | [Install](https://github.com/a-ratz/tare/blob/main/docs/install.md) · [The words tare uses](https://github.com/a-ratz/tare/blob/main/docs/words.md) · [Running agents in rooms](https://github.com/a-ratz/tare/blob/main/docs/running-agents.md) · [Reading a probe](https://github.com/a-ratz/tare/blob/main/docs/probe.md) |
|
|
115
|
+
| **Measuring** | [Overview](https://github.com/a-ratz/tare/blob/main/docs/measuring.md) · [Calibrate](https://github.com/a-ratz/tare/blob/main/docs/calibrate.md) · [Judge](https://github.com/a-ratz/tare/blob/main/docs/judge.md) · [Swap](https://github.com/a-ratz/tare/blob/main/docs/swap.md) · [Cliff](https://github.com/a-ratz/tare/blob/main/docs/cliff.md) · [Tokens and cost](https://github.com/a-ratz/tare/blob/main/docs/tokens-and-cost.md) · [Dashboard and rerun](https://github.com/a-ratz/tare/blob/main/docs/dashboard.md) |
|
|
116
|
+
| **Background** | [How it works](https://github.com/a-ratz/tare/blob/main/docs/how-it-works.md) · [Concept: the design and every measurement](https://github.com/a-ratz/tare/blob/main/CONCEPT.md) · [Experiments](https://github.com/a-ratz/tare/tree/main/experiments) |
|
|
117
|
+
|
|
118
|
+
## Status
|
|
119
|
+
|
|
120
|
+
tare is on PyPI as `tare-cli`. Claude Code, Codex and Pi run on Linux, WSL and macOS; the
|
|
121
|
+
Antigravity CLI runs on Linux and WSL, and stops with a message on macOS for now. What comes next
|
|
122
|
+
is in the [open epics](https://github.com/a-ratz/tare/issues?q=is%3Aissue+is%3Aopen+label%3Atype%3Aepic).
|
|
123
|
+
|
|
124
|
+
## Contributing
|
|
125
|
+
|
|
126
|
+
Every change starts as an issue. Larger work is an epic, split into features, and one epic
|
|
127
|
+
becomes one branch and one pull request. [AGENTS.md](https://github.com/a-ratz/tare/blob/main/AGENTS.md)
|
|
128
|
+
describes the process, the checks and the conventions. Run the tests with `uv run pytest`.
|
|
129
|
+
|
|
130
|
+
## License
|
|
131
|
+
|
|
132
|
+
MIT. See [LICENSE](https://github.com/a-ratz/tare/blob/main/LICENSE).
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
Crafted with [Readme Craft](https://github.com/motiful/readme-craft)
|
|
137
|
+
|
|
138
|
+
[license-shield]: https://img.shields.io/badge/license-MIT-blue.svg
|
|
139
|
+
[license-url]: https://github.com/a-ratz/tare/blob/main/LICENSE
|
|
140
|
+
[version-shield]: https://img.shields.io/pypi/v/tare-cli.svg
|
|
141
|
+
[version-url]: https://pypi.org/project/tare-cli/
|
|
142
|
+
[python-shield]: https://img.shields.io/badge/python-3.11%2B-3776AB.svg
|
|
143
|
+
[python-url]: https://www.python.org/
|
|
144
|
+
[platform-shield]: https://img.shields.io/badge/platform-Linux%20%7C%20WSL%20%7C%20macOS-555555.svg
|
|
145
|
+
[platform-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
|
146
|
+
[agent-shield]: https://img.shields.io/badge/agents-Claude%20Code%20%7C%20Codex%20%7C%20Pi%20%7C%20Antigravity-D97757.svg
|
|
147
|
+
[agent-url]: https://github.com/a-ratz/tare/blob/main/docs/install.md
|
|
@@ -15,6 +15,7 @@ import stat
|
|
|
15
15
|
import subprocess
|
|
16
16
|
import sys
|
|
17
17
|
import tempfile
|
|
18
|
+
import time
|
|
18
19
|
from contextlib import contextmanager
|
|
19
20
|
from dataclasses import dataclass, field
|
|
20
21
|
from pathlib import Path
|
|
@@ -28,8 +29,11 @@ SEATBELT_ROOMS = "/private/tmp"
|
|
|
28
29
|
# Copied from the caller when set; everything else stays outside.
|
|
29
30
|
PASS_ENV = ("TERM", "COLORTERM", "LANG", "LC_ALL")
|
|
30
31
|
# The room must not refresh the login: a refresh may rotate the token family the real
|
|
31
|
-
# session depends on (CONCEPT.md, layer 2). An hour covers ordinary runs
|
|
32
|
+
# session depends on (CONCEPT.md, layer 2). An hour covers ordinary runs; a shorter run may
|
|
33
|
+
# ask for less, but never so little that the copy starts close to its renewal.
|
|
32
34
|
MIN_TOKEN_LIFETIME = 3600
|
|
35
|
+
MIN_LOGIN_ENV = "TARE_MIN_LOGIN_MINUTES"
|
|
36
|
+
MIN_LOGIN_FLOOR = 10 # minutes
|
|
33
37
|
|
|
34
38
|
|
|
35
39
|
class TareError(Exception):
|
|
@@ -61,11 +65,26 @@ class Room:
|
|
|
61
65
|
return f"{self.inside_home}/{agent.config_dir}"
|
|
62
66
|
|
|
63
67
|
|
|
68
|
+
def _min_lifetime() -> int:
|
|
69
|
+
"""Seconds the login copy must still hold: an hour, or what TARE_MIN_LOGIN_MINUTES asks for."""
|
|
70
|
+
minutes = os.environ.get(MIN_LOGIN_ENV)
|
|
71
|
+
if minutes is None:
|
|
72
|
+
return MIN_TOKEN_LIFETIME
|
|
73
|
+
if not minutes.isdigit() or int(minutes) < MIN_LOGIN_FLOOR:
|
|
74
|
+
raise TareError(f"{MIN_LOGIN_ENV} must be a whole number of minutes, at least {MIN_LOGIN_FLOOR}")
|
|
75
|
+
return int(minutes) * 60
|
|
76
|
+
|
|
77
|
+
|
|
64
78
|
def _check_login(agent, real):
|
|
65
79
|
left = agent.token_lifetime(real)
|
|
66
|
-
if left <
|
|
67
|
-
|
|
68
|
-
|
|
80
|
+
if left < _min_lifetime():
|
|
81
|
+
# the agent renews only close to expiry, so a session started earlier changes nothing
|
|
82
|
+
# with the zone: on the night the clocks go back, a bare 02:04 reads as already past
|
|
83
|
+
minutes, at = max(0, int(left // 60)), time.strftime("%H:%M %Z", time.localtime(time.time() + left)).strip()
|
|
84
|
+
shorter = (f"; a run that ends well before then can start now with {MIN_LOGIN_ENV}={minutes} or less"
|
|
85
|
+
if minutes >= MIN_LOGIN_FLOOR else "")
|
|
86
|
+
raise TareError(f"the {agent.name} login token expires in {minutes} min, at {at}: to renew it, start "
|
|
87
|
+
f"{agent.name} outside tare after {at}, then retry{shorter}")
|
|
69
88
|
|
|
70
89
|
|
|
71
90
|
def room_env(agent, room: Room | None = None, extra: dict[str, str] | None = None) -> dict[str, str]:
|
|
@@ -60,6 +60,35 @@ def test_room_refuses_a_login_that_would_need_a_refresh(tmp_path, make, agent):
|
|
|
60
60
|
pass
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
def test_refusal_names_the_expiry_and_how_a_shorter_run_starts(tmp_path):
|
|
64
|
+
real = claude_real(tmp_path, expires_in=1800)
|
|
65
|
+
with pytest.raises(TareError, match=r"expires in 29 min, at \d\d:\d\d.*: to renew it, start claude outside "
|
|
66
|
+
r"tare after \d\d:\d\d.*, then retry; .* TARE_MIN_LOGIN_MINUTES=29 or less"):
|
|
67
|
+
with room_home(Claude(), real):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_no_shorter_run_is_offered_below_the_floor(tmp_path):
|
|
72
|
+
with pytest.raises(TareError) as refused:
|
|
73
|
+
with room_home(Claude(), claude_real(tmp_path, expires_in=300)):
|
|
74
|
+
pass
|
|
75
|
+
assert "TARE_MIN_LOGIN_MINUTES" not in str(refused.value)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_a_shorter_run_starts_with_less_than_an_hour_left(tmp_path, monkeypatch):
|
|
79
|
+
monkeypatch.setenv("TARE_MIN_LOGIN_MINUTES", "15")
|
|
80
|
+
with room_home(Claude(), claude_real(tmp_path, expires_in=1200)) as home:
|
|
81
|
+
assert (home / ".claude-config" / ".credentials.json").exists()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@pytest.mark.parametrize("minutes", ["9", "0", "-5", "ten", ""])
|
|
85
|
+
def test_the_minimum_never_drops_below_the_floor(tmp_path, monkeypatch, minutes):
|
|
86
|
+
monkeypatch.setenv("TARE_MIN_LOGIN_MINUTES", minutes)
|
|
87
|
+
with pytest.raises(TareError, match="TARE_MIN_LOGIN_MINUTES must be a whole number of minutes, at least 10"):
|
|
88
|
+
with room_home(Claude(), claude_real(tmp_path)):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
|
|
63
92
|
def test_codex_api_key_login_never_expires(tmp_path):
|
|
64
93
|
real = codex_real(tmp_path)
|
|
65
94
|
(real.config / "auth.json").write_text(json.dumps({"OPENAI_API_KEY": "sk-test"}))
|