tare-cli 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tare_cli-0.4.0 → tare_cli-0.4.1}/CONCEPT.md +11 -2
- {tare_cli-0.4.0 → tare_cli-0.4.1}/PKG-INFO +1 -1
- {tare_cli-0.4.0 → tare_cli-0.4.1}/pyproject.toml +1 -1
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/agents.py +18 -1
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/room.py +14 -0
- tare_cli-0.4.1/tests/test_agents.py +55 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_room.py +29 -0
- tare_cli-0.4.0/tests/test_agents.py +0 -9
- {tare_cli-0.4.0 → tare_cli-0.4.1}/.gitignore +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/LICENSE +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/README.md +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/__init__.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/answer.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/fonts/IBMPlexMono-Regular.woff2 +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/fonts/IBMPlexMono-SemiBold.woff2 +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/fonts/IBMPlexSans-Regular.woff2 +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/fonts/IBMPlexSans-SemiBold.woff2 +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/fonts/OFL.txt +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/logo-dark.svg +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/logo-light.svg +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/assets/tare.css +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/calibrate.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/capsule.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/cli.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/cliff.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/dashboard.html +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/dashboard.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/fake.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/journal.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/judge.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/overview.html +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/probe.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/recipe.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/swap.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/trail.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/src/tare/usage.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/conftest.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_agy.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_answer.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_calibrate.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_cli.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_cliff.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_dashboard.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_fake.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_judge.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_pi.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_probe.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_seatbelt.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_swap.py +0 -0
- {tare_cli-0.4.0 → tare_cli-0.4.1}/tests/test_usage.py +0 -0
|
@@ -98,7 +98,10 @@ A bubblewrap room is a sandbox for one agent run: an empty `/home` with the room
|
|
|
98
98
|
directory at `/home/tare`, a fresh copy of the login, the agent's executable, the project at
|
|
99
99
|
`/work`, and the environment allowlist. `tare claude` and the other agent commands mount your
|
|
100
100
|
project directly, so the agent changes your real files. Calibrate, Swap and Cliff give every run
|
|
101
|
-
its own copy of the project. The room has network access and a working login.
|
|
101
|
+
its own copy of the project. The room has network access and a working login. Where `/etc/resolv.conf` is a link into `/run`
|
|
102
|
+
(systemd-resolved, NetworkManager), the room mounts the one file the link names; without it no host
|
|
103
|
+
name would resolve in the room, and the probe would not notice, because it only talks to the fake
|
|
104
|
+
model server by address. It keeps your setup out of the measurement. It does not protect your machine from a hostile
|
|
102
105
|
agent.
|
|
103
106
|
|
|
104
107
|
A Seatbelt room (macOS) cannot mount anything. It is one directory under `/private/tmp` that
|
|
@@ -156,6 +159,10 @@ reach.
|
|
|
156
159
|
Each agent needs its own way to the fake model server, its own login copy and its own markers:
|
|
157
160
|
lines of your own files that the probe looks for in a request.
|
|
158
161
|
|
|
162
|
+
tare mounts the agent's own program into the room. Where the name on `PATH` is a mise shim, a link
|
|
163
|
+
to mise that picks the tool by the name it was called as, tare asks mise for the program behind it
|
|
164
|
+
(`mise which`). Mounted in its place, mise would find no installed tool in the room.
|
|
165
|
+
|
|
159
166
|
| Agent | How the room is set up | Way to the fake model server | Kinds of context the control run showed on the measuring machine |
|
|
160
167
|
|---|---|---|---|
|
|
161
168
|
| Claude Code | `CLAUDE_CONFIG_DIR` and `HOME` in the room, a copy of `.credentials.json`, `--strict-mcp-config --setting-sources project,local` | `ANTHROPIC_BASE_URL` | instructions, home path, MCP servers, skills, plugins, email, reach |
|
|
@@ -300,7 +307,8 @@ names what has changed since.
|
|
|
300
307
|
## Measurements
|
|
301
308
|
|
|
302
309
|
All measurements ran on WSL2 on 2026-10-04 and 2026-10-05. The rows that name macOS ran on
|
|
303
|
-
macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
310
|
+
macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06. The row that names Omarchy ran on native Linux on
|
|
311
|
+
2026-11-16.
|
|
304
312
|
|
|
305
313
|
| What | Agent and version | Result | Source |
|
|
306
314
|
|---|---|---|---|
|
|
@@ -312,6 +320,7 @@ macOS 27.0 (arm64) on 2026-10-05 and 2026-10-06.
|
|
|
312
320
|
| Scripted Cliff and Swap | Claude Code against scripted models | Cliff found step 4. Swap passed the start check and found the blame passing from the agent to the workspace between the handovers at 0 and 0.5 | `experiments/cliff-scripted`, `experiments/swap-scripted` |
|
|
313
321
|
| Real task 1: a data migration | Claude Code with Sonnet and Haiku, Codex | Deleting the source file after a wrong-encoding import was planned as the point of no return. Swap showed that the workspace could still be repaired. Haiku passed about 1 in 14 runs, Codex with gpt-6.1-sol 4 of 4 | [results](experiments/migration/RESULTS.md) |
|
|
314
322
|
| Real task 2: a web page judged by Sonnet | Claude Code with Haiku, Codex with gpt-6.1-sol | Swap confirmed all four predictions of the plan written before the runs. Cliff called a failed run unlucky, and a later rescoring of the same page confirmed it | [results](experiments/html/RESULTS.md) |
|
|
323
|
+
| Native Linux with systemd-resolved: a room that resolves host names | Claude Code 2.1.325, bubblewrap 0.13.1, Omarchy 4.5.0 (Arch Linux) | before the fix the room read `tare: 0.00` and the agent's first request failed with a DNS error; with the resolver file mounted the probe still read `tare: 0.00` and the agent answered one prompt | [#152](https://github.com/a-ratz/tare/issues/152) |
|
|
315
324
|
| Field test: two ideation skills compared | Claude Code 2.1.289 with Opus 5.5 | all 18 rooms read `tare: 0.00`, including a skill that started `claude -p` inside the room | not published |
|
|
316
325
|
| macOS: what a Seatbelt room has to handle | Claude Code 2.1.289, Codex CLI 0.159.0, Pi 0.99.1 | Claude Code's login is in the Keychain. With only the home directory denied, `/var/folders`, `/private/tmp` and the pasteboard stayed readable. Agents' own sandboxes fail inside a room (`sandbox_apply: Operation not permitted`) | comments on epic #85 |
|
|
317
326
|
| macOS: a Seatbelt room with the real login | same | each agent answered one prompt. The room and its login copy were removed afterwards, and the Keychain item was unchanged (digest before and after). `tare probe` read `tare: 0.00` for all three | comments on epic #85 |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tare-cli
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Prove your coding agent starts without your personal setup before you measure it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/a-ratz/tare
|
|
6
6
|
Project-URL: Repository, https://github.com/a-ratz/tare
|
|
@@ -68,7 +68,24 @@ def _which(name: str) -> Path:
|
|
|
68
68
|
found = shutil.which(name)
|
|
69
69
|
if not found:
|
|
70
70
|
raise TareError(f"{name} is not on PATH")
|
|
71
|
-
|
|
71
|
+
real = Path(found).resolve()
|
|
72
|
+
if real.name == "mise" and Path(found).name != "mise":
|
|
73
|
+
return _behind_mise(name, found, real)
|
|
74
|
+
return real
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _behind_mise(name: str, shim: str, mise: Path) -> Path:
|
|
78
|
+
"""The program behind a mise shim.
|
|
79
|
+
|
|
80
|
+
A mise shim is a link to mise itself, which picks the tool by the name it was called as. Mounted into
|
|
81
|
+
the room in place of the agent, mise finds no installed tool there and tries to fetch one.
|
|
82
|
+
"""
|
|
83
|
+
out = subprocess.run([str(mise), "which", name], capture_output=True, text=True)
|
|
84
|
+
tool = Path(out.stdout.strip()) if out.returncode == 0 and out.stdout.strip() else None
|
|
85
|
+
if tool is None or not tool.is_file():
|
|
86
|
+
raise TareError(f"{name} on PATH is a mise shim ({shim}), and `mise which {name}` did not name the program. "
|
|
87
|
+
f"Put the directory of the program itself first on PATH.")
|
|
88
|
+
return tool.resolve()
|
|
72
89
|
|
|
73
90
|
|
|
74
91
|
def _absolute_links(original: Path, copy: Path):
|
|
@@ -123,6 +123,7 @@ class Bubblewrap:
|
|
|
123
123
|
if Path("/mnt/wsl").is_dir():
|
|
124
124
|
# WSL keeps the resolver behind a symlink into /mnt/wsl; without it DNS fails.
|
|
125
125
|
argv += ["--ro-bind", "/mnt/wsl", "/mnt/wsl"]
|
|
126
|
+
argv += _resolver()
|
|
126
127
|
run = ["--bind", str(room.run), room.inside_run] if room.run else []
|
|
127
128
|
argv += ["--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--tmpfs", "/home",
|
|
128
129
|
"--bind", str(room.home), room.inside_home, *binds, *run, *(extra_binds or []),
|
|
@@ -176,6 +177,19 @@ class Seatbelt:
|
|
|
176
177
|
|
|
177
178
|
|
|
178
179
|
@functools.cache
|
|
180
|
+
def _resolver(conf: Path = Path("/etc/resolv.conf")) -> list[str]:
|
|
181
|
+
"""The bind for a resolver file that sits outside the room's mounts.
|
|
182
|
+
|
|
183
|
+
systemd-resolved, NetworkManager and resolvconf keep /etc/resolv.conf as a symlink into /run. The room
|
|
184
|
+
mounts /etc but not /run, so the link would point nowhere and no host name would resolve. Only the file
|
|
185
|
+
the link names is mounted, not its directory.
|
|
186
|
+
"""
|
|
187
|
+
target = conf.resolve()
|
|
188
|
+
if not conf.is_symlink() or not target.is_file() or str(target).startswith(("/etc/", "/usr/", "/mnt/wsl/")):
|
|
189
|
+
return []
|
|
190
|
+
return ["--ro-bind", str(target), str(target)]
|
|
191
|
+
|
|
192
|
+
|
|
179
193
|
def _user_dir() -> Path:
|
|
180
194
|
"""The user's directory under /var/folders: its T and C hold other programs' temporary files and caches."""
|
|
181
195
|
out = subprocess.run(["getconf", "DARWIN_USER_DIR"], capture_output=True, text=True, check=True).stdout
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from tare import agents
|
|
4
|
+
from tare.agents import AGENTS
|
|
5
|
+
from tare.room import TareError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_a_resumed_claude_tail_streams_events_like_a_fresh_start():
|
|
9
|
+
claude = AGENTS["claude"]
|
|
10
|
+
fresh = claude.run_args("task", [])
|
|
11
|
+
resumed = claude.resume_args("session", "Continue.", [])
|
|
12
|
+
for flag in ("--output-format", "stream-json", "--verbose"):
|
|
13
|
+
assert flag in fresh and flag in resumed
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def mise_shim(tmp_path, script):
|
|
17
|
+
"""A stand-in for mise and a shim that links to it, the way `mise` sets up `claude` on PATH."""
|
|
18
|
+
mise = tmp_path / "bin/mise"
|
|
19
|
+
mise.parent.mkdir()
|
|
20
|
+
mise.write_text(f"#!/bin/sh\n{script}\n")
|
|
21
|
+
mise.chmod(0o755)
|
|
22
|
+
shim = tmp_path / "shims/claude"
|
|
23
|
+
shim.parent.mkdir()
|
|
24
|
+
shim.symlink_to(mise)
|
|
25
|
+
return mise, shim
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_which_finds_the_program_behind_a_mise_shim(tmp_path, monkeypatch):
|
|
29
|
+
tool = tmp_path / "installs/claude/2.1.325/claude"
|
|
30
|
+
tool.parent.mkdir(parents=True)
|
|
31
|
+
tool.write_text("")
|
|
32
|
+
_, shim = mise_shim(tmp_path, f'[ "$1" = which ] && [ "$2" = claude ] && echo "{tool}"')
|
|
33
|
+
monkeypatch.setattr(agents.shutil, "which", lambda name: str(shim))
|
|
34
|
+
assert agents._which("claude") == tool
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_which_names_the_shim_when_mise_does_not_know_the_program(tmp_path, monkeypatch):
|
|
38
|
+
_, shim = mise_shim(tmp_path, "echo 'mise ERROR claude is not installed' >&2; exit 1")
|
|
39
|
+
monkeypatch.setattr(agents.shutil, "which", lambda name: str(shim))
|
|
40
|
+
with pytest.raises(TareError, match="mise shim .*first on PATH"):
|
|
41
|
+
agents._which("claude")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_which_leaves_mise_itself_and_other_links_alone(tmp_path, monkeypatch):
|
|
45
|
+
mise, _ = mise_shim(tmp_path, "exit 1")
|
|
46
|
+
monkeypatch.setattr(agents.shutil, "which", lambda name: str(mise))
|
|
47
|
+
assert agents._which("mise") == mise
|
|
48
|
+
program = tmp_path / "opt/claude"
|
|
49
|
+
program.parent.mkdir()
|
|
50
|
+
program.write_text("")
|
|
51
|
+
link = tmp_path / "link/claude"
|
|
52
|
+
link.parent.mkdir()
|
|
53
|
+
link.symlink_to(program)
|
|
54
|
+
monkeypatch.setattr(agents.shutil, "which", lambda name: str(link))
|
|
55
|
+
assert agents._which("claude") == program
|
|
@@ -103,6 +103,35 @@ def test_room_refuses_without_login(tmp_path, agent):
|
|
|
103
103
|
pass
|
|
104
104
|
|
|
105
105
|
|
|
106
|
+
def test_a_resolver_file_behind_a_symlink_is_mounted_alone(tmp_path):
|
|
107
|
+
# systemd-resolved: /etc/resolv.conf -> ../run/systemd/resolve/stub-resolv.conf
|
|
108
|
+
target = tmp_path / "run/systemd/resolve/stub-resolv.conf"
|
|
109
|
+
target.parent.mkdir(parents=True)
|
|
110
|
+
target.write_text("nameserver 127.0.0.53\n")
|
|
111
|
+
(tmp_path / "run/systemd/resolve/resolv.conf").write_text("nameserver 192.0.2.1\n")
|
|
112
|
+
conf = tmp_path / "etc/resolv.conf"
|
|
113
|
+
conf.parent.mkdir()
|
|
114
|
+
conf.symlink_to(target)
|
|
115
|
+
assert room._resolver(conf) == ["--ro-bind", str(target), str(target)]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_a_plain_resolver_file_or_a_dangling_link_needs_no_mount(tmp_path):
|
|
119
|
+
plain = tmp_path / "resolv.conf"
|
|
120
|
+
plain.write_text("nameserver 192.0.2.1\n")
|
|
121
|
+
assert room._resolver(plain) == []
|
|
122
|
+
dangling = tmp_path / "dangling.conf"
|
|
123
|
+
dangling.symlink_to(tmp_path / "gone")
|
|
124
|
+
assert room._resolver(dangling) == []
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_bwrap_mounts_the_resolver_file(tmp_path, monkeypatch):
|
|
128
|
+
monkeypatch.setattr(room, "_resolver", lambda: ["--ro-bind", "/run/r/resolv.conf", "/run/r/resolv.conf"])
|
|
129
|
+
argv = bwrap(Claude(), claude_real(tmp_path), tmp_path, tmp_path, ["claude", "-p"], {})
|
|
130
|
+
at = argv.index("/run/r/resolv.conf")
|
|
131
|
+
assert argv[at - 1:at + 2] == ["--ro-bind", "/run/r/resolv.conf", "/run/r/resolv.conf"]
|
|
132
|
+
assert at < argv.index("--tmpfs") # mounted with /usr and /etc, before the room's own home
|
|
133
|
+
|
|
134
|
+
|
|
106
135
|
def test_bwrap_clears_the_environment_down_to_the_allowlist(tmp_path, monkeypatch):
|
|
107
136
|
monkeypatch.setenv("SOME_API_KEY", "x")
|
|
108
137
|
monkeypatch.setenv("TERM", "xterm")
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
from tare.agents import AGENTS
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
def test_a_resumed_claude_tail_streams_events_like_a_fresh_start():
|
|
5
|
-
claude = AGENTS["claude"]
|
|
6
|
-
fresh = claude.run_args("task", [])
|
|
7
|
-
resumed = claude.resume_args("session", "Continue.", [])
|
|
8
|
-
for flag in ("--output-format", "stream-json", "--verbose"):
|
|
9
|
-
assert flag in fresh and flag in resumed
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|