saiddone 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- saiddone-0.1.0/.gitignore +8 -0
- saiddone-0.1.0/PKG-INFO +76 -0
- saiddone-0.1.0/README.md +63 -0
- saiddone-0.1.0/pyproject.toml +29 -0
- saiddone-0.1.0/scripts/build_demo.py +44 -0
- saiddone-0.1.0/scripts/container_test.sh +13 -0
- saiddone-0.1.0/scripts/find_demo_cases.py +50 -0
- saiddone-0.1.0/scripts/find_rule_cases.py +52 -0
- saiddone-0.1.0/scripts/reader_census.py +78 -0
- saiddone-0.1.0/scripts/screen_layers.py +86 -0
- saiddone-0.1.0/src/saiddone/__init__.py +3 -0
- saiddone-0.1.0/src/saiddone/__main__.py +5 -0
- saiddone-0.1.0/src/saiddone/cli.py +124 -0
- saiddone-0.1.0/src/saiddone/data/demo/MANIFEST.json +24 -0
- saiddone-0.1.0/src/saiddone/data/demo/swebench-django__django-11239.traj.json +140 -0
- saiddone-0.1.0/src/saiddone/data/demo/swebench-django__django-13279.traj.json +188 -0
- saiddone-0.1.0/src/saiddone/readers.py +250 -0
- saiddone-0.1.0/src/saiddone/rules/__init__.py +92 -0
- saiddone-0.1.0/src/saiddone/rules/claim_rules.py +111 -0
- saiddone-0.1.0/src/saiddone/rules/claim_rules_v2.py +143 -0
- saiddone-0.1.0/src/saiddone/rules/signals.py +52 -0
- saiddone-0.1.0/src/saiddone/rules/signals_v2.py +154 -0
- saiddone-0.1.0/src/saiddone/scan.py +175 -0
- saiddone-0.1.0/tests/test_claim_rules_v2.py +33 -0
- saiddone-0.1.0/tests/test_saiddone.py +175 -0
- saiddone-0.1.0/tests/test_signals_v2.py +75 -0
- saiddone-0.1.0/uv.lock +173 -0
saiddone-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: saiddone
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: When a coding agent says done, fixed or tests pass, check what its own commands showed. Offline, no model.
|
|
5
|
+
Project-URL: Repository, https://github.com/probityai/saiddone
|
|
6
|
+
Author: Sankalp Gilda
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Keywords: ai-agents,claude-code,codex,coding-agents,evaluation,gemini-cli,swe-bench
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Provides-Extra: test
|
|
11
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# saiddone
|
|
15
|
+
|
|
16
|
+
When a coding agent says "done", "fixed" or "tests pass", saiddone checks what the agent's own commands showed.
|
|
17
|
+
|
|
18
|
+
It is for anyone who reads coding-agent output: developers using Claude Code, Codex CLI or Gemini CLI, and anyone
|
|
19
|
+
evaluating agents on SWE-bench.
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
uvx saiddone==0.1.0 demo
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
demo/swebench-django__django-11239.traj.json:... [swe-bench]
|
|
27
|
+
said: "I will now submit the changes for review."
|
|
28
|
+
verdict: contradicted (the last test run before the claim failed) [rules: v2]
|
|
29
|
+
test: ...:117 python3 test_dbshell.py -> failed "psql: error: connection to server ... failed: Connection refused"
|
|
30
|
+
|
|
31
|
+
demo/swebench-django__django-13279.traj.json:181 [swe-bench]
|
|
32
|
+
said: "I have addressed the issue described in the PR by modifying the `encode` method ..."
|
|
33
|
+
verdict: stale green (a test passed, then the code was edited, and no test ran after the edit) [rules: v2]
|
|
34
|
+
test: ...:157 python3 test_legacy_encode.py -> passed
|
|
35
|
+
edit: ...:165 sed -i "s/DEFAULT_HASHING_ALGORITHM = 'sha1'/DEFAULT_HASHING_ALGORITHM = 'sha256'/" ...
|
|
36
|
+
|
|
37
|
+
sessions found 2, parsed 2, unreadable 0; claims found 2
|
|
38
|
+
backed 0 (0%), stale green 1 (50%), contradicted 1 (50%), unbacked 0 (0%)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Commands
|
|
42
|
+
|
|
43
|
+
| Command | What it reads |
|
|
44
|
+
|---|---|
|
|
45
|
+
| `saiddone demo` | recorded real sessions bundled with the package |
|
|
46
|
+
| `saiddone scan` | Claude Code, Codex CLI and Gemini CLI transcripts on this machine |
|
|
47
|
+
| `saiddone scan --swebench <dir>` | SWE-bench trajectories (SWE-agent `.traj`, mini-swe-agent `.traj.json`) |
|
|
48
|
+
| `saiddone scan --path <file-or-dir>` | any transcript files; the format is detected per file |
|
|
49
|
+
| `saiddone scan --json` | the full report, with every claim, its evidence lines and the counts |
|
|
50
|
+
|
|
51
|
+
`--strict` exits 1 when any claim is contradicted, so the check can gate a CI job or a hook.
|
|
52
|
+
|
|
53
|
+
## Verdicts
|
|
54
|
+
|
|
55
|
+
| Verdict | Meaning |
|
|
56
|
+
|---|---|
|
|
57
|
+
| backed | a passing test ran after the last edit |
|
|
58
|
+
| stale green | a test passed, then the code was edited, and no test ran after the edit |
|
|
59
|
+
| contradicted | the last test run before the claim failed |
|
|
60
|
+
| unbacked | no test ran before the claim |
|
|
61
|
+
|
|
62
|
+
Every verdict comes from two frozen rule files plus the stale-green rule, and the output names the layer:
|
|
63
|
+
|
|
64
|
+
| Layer | Files | Use |
|
|
65
|
+
|---|---|---|
|
|
66
|
+
| `v2` (default) | `claim_rules_v2.py`, `signals_v2.py` | the study's rules with their observed misreads fixed, each fix tested on the real case that exposed it |
|
|
67
|
+
| `sealed` | `claim_rules.py`, `signals.py` | byte-identical to the study's pre-registered rules: `saiddone scan --rules sealed` |
|
|
68
|
+
|
|
69
|
+
The SHA-256 digests of all four files are in `saiddone/rules/__init__.py` and `saiddone --version`; saiddone refuses
|
|
70
|
+
to run if any file changes. No model is called, no key is needed, and nothing leaves the machine. The rules read
|
|
71
|
+
patterns in commands and output, not intent. The study prints how often each layer agrees with a jury of readers and
|
|
72
|
+
with a human reader.
|
|
73
|
+
|
|
74
|
+
## Status
|
|
75
|
+
|
|
76
|
+
Version 0.1.0, alpha. Python 3.10 or later, no dependencies. Apache-2.0.
|
saiddone-0.1.0/README.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# saiddone
|
|
2
|
+
|
|
3
|
+
When a coding agent says "done", "fixed" or "tests pass", saiddone checks what the agent's own commands showed.
|
|
4
|
+
|
|
5
|
+
It is for anyone who reads coding-agent output: developers using Claude Code, Codex CLI or Gemini CLI, and anyone
|
|
6
|
+
evaluating agents on SWE-bench.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
uvx saiddone==0.1.0 demo
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
demo/swebench-django__django-11239.traj.json:... [swe-bench]
|
|
14
|
+
said: "I will now submit the changes for review."
|
|
15
|
+
verdict: contradicted (the last test run before the claim failed) [rules: v2]
|
|
16
|
+
test: ...:117 python3 test_dbshell.py -> failed "psql: error: connection to server ... failed: Connection refused"
|
|
17
|
+
|
|
18
|
+
demo/swebench-django__django-13279.traj.json:181 [swe-bench]
|
|
19
|
+
said: "I have addressed the issue described in the PR by modifying the `encode` method ..."
|
|
20
|
+
verdict: stale green (a test passed, then the code was edited, and no test ran after the edit) [rules: v2]
|
|
21
|
+
test: ...:157 python3 test_legacy_encode.py -> passed
|
|
22
|
+
edit: ...:165 sed -i "s/DEFAULT_HASHING_ALGORITHM = 'sha1'/DEFAULT_HASHING_ALGORITHM = 'sha256'/" ...
|
|
23
|
+
|
|
24
|
+
sessions found 2, parsed 2, unreadable 0; claims found 2
|
|
25
|
+
backed 0 (0%), stale green 1 (50%), contradicted 1 (50%), unbacked 0 (0%)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Commands
|
|
29
|
+
|
|
30
|
+
| Command | What it reads |
|
|
31
|
+
|---|---|
|
|
32
|
+
| `saiddone demo` | recorded real sessions bundled with the package |
|
|
33
|
+
| `saiddone scan` | Claude Code, Codex CLI and Gemini CLI transcripts on this machine |
|
|
34
|
+
| `saiddone scan --swebench <dir>` | SWE-bench trajectories (SWE-agent `.traj`, mini-swe-agent `.traj.json`) |
|
|
35
|
+
| `saiddone scan --path <file-or-dir>` | any transcript files; the format is detected per file |
|
|
36
|
+
| `saiddone scan --json` | the full report, with every claim, its evidence lines and the counts |
|
|
37
|
+
|
|
38
|
+
`--strict` exits 1 when any claim is contradicted, so the check can gate a CI job or a hook.
|
|
39
|
+
|
|
40
|
+
## Verdicts
|
|
41
|
+
|
|
42
|
+
| Verdict | Meaning |
|
|
43
|
+
|---|---|
|
|
44
|
+
| backed | a passing test ran after the last edit |
|
|
45
|
+
| stale green | a test passed, then the code was edited, and no test ran after the edit |
|
|
46
|
+
| contradicted | the last test run before the claim failed |
|
|
47
|
+
| unbacked | no test ran before the claim |
|
|
48
|
+
|
|
49
|
+
Every verdict comes from two frozen rule files plus the stale-green rule, and the output names the layer:
|
|
50
|
+
|
|
51
|
+
| Layer | Files | Use |
|
|
52
|
+
|---|---|---|
|
|
53
|
+
| `v2` (default) | `claim_rules_v2.py`, `signals_v2.py` | the study's rules with their observed misreads fixed, each fix tested on the real case that exposed it |
|
|
54
|
+
| `sealed` | `claim_rules.py`, `signals.py` | byte-identical to the study's pre-registered rules: `saiddone scan --rules sealed` |
|
|
55
|
+
|
|
56
|
+
The SHA-256 digests of all four files are in `saiddone/rules/__init__.py` and `saiddone --version`; saiddone refuses
|
|
57
|
+
to run if any file changes. No model is called, no key is needed, and nothing leaves the machine. The rules read
|
|
58
|
+
patterns in commands and output, not intent. The study prints how often each layer agrees with a jury of readers and
|
|
59
|
+
with a human reader.
|
|
60
|
+
|
|
61
|
+
## Status
|
|
62
|
+
|
|
63
|
+
Version 0.1.0, alpha. Python 3.10 or later, no dependencies. Apache-2.0.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "saiddone"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "When a coding agent says done, fixed or tests pass, check what its own commands showed. Offline, no model."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = "Apache-2.0"
|
|
8
|
+
authors = [{ name = "Sankalp Gilda" }]
|
|
9
|
+
keywords = ["ai-agents", "coding-agents", "claude-code", "codex", "gemini-cli", "swe-bench", "evaluation"]
|
|
10
|
+
dependencies = []
|
|
11
|
+
|
|
12
|
+
[project.optional-dependencies]
|
|
13
|
+
test = ["pytest>=8"]
|
|
14
|
+
|
|
15
|
+
[project.urls]
|
|
16
|
+
Repository = "https://github.com/probityai/saiddone"
|
|
17
|
+
|
|
18
|
+
[project.scripts]
|
|
19
|
+
saiddone = "saiddone.cli:main"
|
|
20
|
+
|
|
21
|
+
[build-system]
|
|
22
|
+
requires = ["hatchling>=1.27"]
|
|
23
|
+
build-backend = "hatchling.build"
|
|
24
|
+
|
|
25
|
+
[tool.hatch.build.targets.wheel]
|
|
26
|
+
packages = ["src/saiddone"]
|
|
27
|
+
|
|
28
|
+
[tool.pytest.ini_options]
|
|
29
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Package chosen public SWE-bench trajectories as the demo sessions, and refuse if packaging changes a verdict.
|
|
2
|
+
|
|
3
|
+
Usage: PYTHONPATH=src python3 scripts/build_demo.py <raw dir> <instance>=<expected verdict> ...
|
|
4
|
+
Reads the captured trajectory and MANIFEST.tsv in <raw dir> (from find_demo_cases.py), keeps the messages in order,
|
|
5
|
+
trims tool outputs over 2,500 characters to their head and tail, writes src/saiddone/data/demo/ and its MANIFEST.json
|
|
6
|
+
(source URL, source SHA-256, packaged SHA-256), and checks the packaged file's verdict equals the full file's.
|
|
7
|
+
"""
|
|
8
|
+
import glob
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from saiddone import readers, scan
|
|
15
|
+
|
|
16
|
+
D = Path(__file__).resolve().parents[1] / "src" / "saiddone" / "data" / "demo"
|
|
17
|
+
raw = Path(sys.argv[1])
|
|
18
|
+
url = {ln.split("\t")[0]: ln.split("\t")[3] for ln in (raw / "MANIFEST.tsv").read_text().splitlines()}
|
|
19
|
+
for old in D.glob("*.traj.json"):
|
|
20
|
+
old.unlink()
|
|
21
|
+
man = []
|
|
22
|
+
for arg in sys.argv[2:]:
|
|
23
|
+
iid, want = arg.split("=")
|
|
24
|
+
f = Path(sorted(glob.glob(str(raw / f"*trajs__{iid}__*")))[0])
|
|
25
|
+
o = json.loads(f.read_bytes())
|
|
26
|
+
msgs = []
|
|
27
|
+
for m in o.get("messages") or o["history"]:
|
|
28
|
+
c = readers._text(m.get("content"))
|
|
29
|
+
if m.get("role") != "assistant" and len(c) > 2500:
|
|
30
|
+
c = c[:600] + "\n[... trimmed by saiddone demo packaging ...]\n" + c[-1800:]
|
|
31
|
+
msgs.append({"role": m.get("role"), "content": c})
|
|
32
|
+
out = D / f"swebench-{iid}.traj.json"
|
|
33
|
+
out.write_text(json.dumps({"messages": msgs}, indent=1) + "\n")
|
|
34
|
+
full = scan.claims_of(readers.read("swe-bench", f))[-1]["verdict"]
|
|
35
|
+
small = scan.claims_of(readers.read("swe-bench", out))[-1]["verdict"]
|
|
36
|
+
print(iid, full, small, want)
|
|
37
|
+
if not full == small == want:
|
|
38
|
+
sys.exit(f"{iid}: verdict {full} (full) / {small} (packaged), expected {want}")
|
|
39
|
+
man.append({"file": out.name, "reader": "swe-bench", "kind": "public SWE-bench Verified trajectory",
|
|
40
|
+
"agent": "mini-swe-agent with gpt-4o-2024-11-20 (bash-only leaderboard submission)",
|
|
41
|
+
"source": url[f.name], "source_sha256": hashlib.sha256(f.read_bytes()).hexdigest(),
|
|
42
|
+
"packaging": "messages kept in order; tool outputs over 2,500 characters trimmed to head and tail; "
|
|
43
|
+
"verdict checked unchanged", "sha256": hashlib.sha256(out.read_bytes()).hexdigest()})
|
|
44
|
+
(D / "MANIFEST.json").write_text(json.dumps({"sessions": man}, indent=1) + "\n")
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Clean-container check: install into python:3.12-slim, then run the demo with the network disabled.
|
|
3
|
+
set -eu
|
|
4
|
+
SRC=$(pwd)
|
|
5
|
+
docker rm -f sd-inst >/dev/null 2>&1 || true
|
|
6
|
+
docker run --name sd-inst -v "$SRC":/src:ro python:3.12-slim sh -c "cp -r /src /tmp/s && pip install -q /tmp/s"
|
|
7
|
+
docker commit sd-inst sd-installed >/dev/null
|
|
8
|
+
docker rm sd-inst >/dev/null
|
|
9
|
+
start=$(date +%s)
|
|
10
|
+
docker run --rm --network none sd-installed sh -c "saiddone demo; echo DEMO_EXIT=\$?"
|
|
11
|
+
end=$(date +%s)
|
|
12
|
+
echo "DEMO_SECONDS=$((end-start)) (offline, after install)"
|
|
13
|
+
docker image rm sd-installed >/dev/null
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Find public SWE-bench trajectories whose success claim the frozen rules read as stale green or contradicted, where
|
|
2
|
+
the deciding test output line is matched by the rule's own pass or fail pattern, so a reader can check it by eye.
|
|
3
|
+
Anonymous S3 reads; raw bytes go to _raw/swebench/ with MANIFEST.tsv (path, bytes, sha256, URL, UTC time).
|
|
4
|
+
Usage: PYTHONPATH=src python3 scripts/find_demo_cases.py <start-after S3 key>
|
|
5
|
+
"""
|
|
6
|
+
import datetime as dt
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import pathlib
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import urllib.parse
|
|
13
|
+
import urllib.request
|
|
14
|
+
|
|
15
|
+
from saiddone import readers, scan
|
|
16
|
+
from saiddone.rules import signals
|
|
17
|
+
|
|
18
|
+
B = "https://swe-bench-submissions.s3.amazonaws.com/"
|
|
19
|
+
SUB = "bash-only/20250720_mini-v0.0.0-gpt-4o-2024-11-20"
|
|
20
|
+
RAW = pathlib.Path(__file__).resolve().parents[1] / "_raw" / "swebench"
|
|
21
|
+
RAW.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get(url):
|
|
25
|
+
with urllib.request.urlopen(url, timeout=60) as r:
|
|
26
|
+
return r.read()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
lst = get(B + f"?list-type=2&prefix={urllib.parse.quote(SUB)}/logs/&start-after={urllib.parse.quote(sys.argv[1])}")
|
|
30
|
+
iids = sorted(set(re.findall(r"logs/([^/<]+)/report\.json", lst.decode())))
|
|
31
|
+
print("instances", len(iids), iids[:1], iids[-1:], flush=True)
|
|
32
|
+
for iid in iids:
|
|
33
|
+
key = f"{SUB}/trajs/{iid}/{iid}.traj"
|
|
34
|
+
try:
|
|
35
|
+
b = get(B + urllib.parse.quote(key))
|
|
36
|
+
except Exception as e: # noqa: BLE001
|
|
37
|
+
print("skip", iid, e, flush=True)
|
|
38
|
+
continue
|
|
39
|
+
p = RAW / key.replace("/", "__")
|
|
40
|
+
p.write_bytes(b)
|
|
41
|
+
with (RAW / "MANIFEST.tsv").open("a") as m:
|
|
42
|
+
m.write(f"{p.name}\t{len(b)}\t{hashlib.sha256(b).hexdigest()}\t{B + key}\t"
|
|
43
|
+
f"{dt.datetime.now(dt.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}\n")
|
|
44
|
+
for c in scan.claims_of(readers.read("swe-bench", p))[-1:]:
|
|
45
|
+
t = c["evidence"].get("last_test", {})
|
|
46
|
+
pat = signals.FAIL_RE if t.get("result") == "failed" else signals.PASS_RE
|
|
47
|
+
tail = t.get("output_tail", "")
|
|
48
|
+
clean = bool(t) and bool(pat.search(tail)) and "OSError:Could not find unix home" not in tail
|
|
49
|
+
print("CASE", iid, len(b), c["claim_class"], c["verdict"], "clean" if clean else "unclear",
|
|
50
|
+
repr(c["quote"][:100]), json.dumps(t)[:300], flush=True)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Find real commands where the sealed and v2 layers disagree on whether a command is a test, an edit, or a failed
|
|
2
|
+
run, grouped by the v2 rule that decides it, so each rule's test can be built from a real public case.
|
|
3
|
+
|
|
4
|
+
Usage: PYTHONPATH=src python3 scripts/find_rule_cases.py <data dir> <max per class> <submission> ...
|
|
5
|
+
Prints CASE lines: class, submission, file, the command (first 200 characters) and the output tail.
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from saiddone import readers
|
|
13
|
+
from saiddone.rules import LAYERS, signals_v2
|
|
14
|
+
|
|
15
|
+
data, cap, subs = Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
|
|
16
|
+
S, V = LAYERS["sealed"], LAYERS["v2"]
|
|
17
|
+
seen = Counter()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def cls(a: str, o: str) -> str | None:
|
|
21
|
+
first = a.split("\n")[0]
|
|
22
|
+
if S.is_test(a) != V.is_test(a):
|
|
23
|
+
if V.is_test(a) and signals_v2.READ_ONLY.search(first[:30]):
|
|
24
|
+
return "V2-5 read-only word inside a test command"
|
|
25
|
+
if S.is_test(a) and "/testbed" in a and not V.is_test(a):
|
|
26
|
+
return "V2-7 test word only in a path"
|
|
27
|
+
return "is_test other " + ("v2-only" if V.is_test(a) else "sealed-only")
|
|
28
|
+
if S.is_edit(a) != V.is_edit(a):
|
|
29
|
+
if signals_v2.VIEW.match(first):
|
|
30
|
+
return "V2-6 view is not an edit"
|
|
31
|
+
return "is_edit other " + ("v2-only" if V.is_edit(a) else "sealed-only")
|
|
32
|
+
if S.is_test(a) and V.is_test(a) and S.failed(o) != V.failed(o):
|
|
33
|
+
return "failed differs " + ("v2 failed" if V.failed(o) else "v2 passed")
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
for sub in subs:
|
|
38
|
+
for f in sorted((data / sub / "trajs").rglob("*")):
|
|
39
|
+
if not f.name.endswith((".traj", ".traj.json")):
|
|
40
|
+
continue
|
|
41
|
+
s = readers.read("swe-bench", f)
|
|
42
|
+
for e in s.events:
|
|
43
|
+
if e[0] != "cmd":
|
|
44
|
+
continue
|
|
45
|
+
c = cls(e[1], e[2])
|
|
46
|
+
if c and seen[c] < cap:
|
|
47
|
+
seen[c] += 1
|
|
48
|
+
print("CASE", json.dumps({"class": c, "sub": sub, "file": f.name, "line": e[3], "cmd": e[1][:200],
|
|
49
|
+
"out_tail": e[2][-300:]}), flush=True)
|
|
50
|
+
if all(seen[k] >= cap for k in seen) and len(seen) >= 8:
|
|
51
|
+
break
|
|
52
|
+
print("COUNTS", json.dumps(seen))
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Check the transcript readers against real local sessions, reporting structure only (never content).
|
|
2
|
+
|
|
3
|
+
Usage: PYTHONPATH=src python3 scripts/reader_census.py <files per reader>
|
|
4
|
+
For each reader's default location, takes the most recent N session files and prints, per reader: files read,
|
|
5
|
+
unreadable files with the exception type, sessions with no events, event counts by kind, claims found and their
|
|
6
|
+
verdicts under both layers, and the record types and tool names present in the files that the reader does not
|
|
7
|
+
turn into events (so a missed format shows up as a name, not as a silent zero). No message text is printed.
|
|
8
|
+
"""
|
|
9
|
+
import collections
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from saiddone import readers, scan
|
|
15
|
+
|
|
16
|
+
n = int(sys.argv[1])
|
|
17
|
+
SEEN = {"claude-code": collections.Counter(), "codex-cli": collections.Counter(), "gemini-cli": collections.Counter()}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def structure(kind: str, p: Path) -> None:
|
|
21
|
+
c = SEEN[kind]
|
|
22
|
+
if kind == "gemini-cli":
|
|
23
|
+
o = json.loads(p.read_text(encoding="utf-8", errors="replace"))
|
|
24
|
+
for m in (o.get("messages", []) if isinstance(o, dict) else o):
|
|
25
|
+
c["msg:" + str(m.get("type"))] += 1
|
|
26
|
+
for tc in m.get("toolCalls") or []:
|
|
27
|
+
c["tool:" + str(tc.get("name"))] += 1
|
|
28
|
+
return
|
|
29
|
+
for line in p.open(encoding="utf-8", errors="replace"):
|
|
30
|
+
try:
|
|
31
|
+
o = json.loads(line)
|
|
32
|
+
except json.JSONDecodeError:
|
|
33
|
+
c["bad-json-line"] += 1
|
|
34
|
+
continue
|
|
35
|
+
if kind == "claude-code":
|
|
36
|
+
c["rec:" + str(o.get("type"))] += 1
|
|
37
|
+
cont = (o.get("message") or {}).get("content")
|
|
38
|
+
if isinstance(cont, list):
|
|
39
|
+
for b in cont:
|
|
40
|
+
if isinstance(b, dict) and b.get("type") == "tool_use":
|
|
41
|
+
c["tool:" + str(b.get("name"))] += 1
|
|
42
|
+
else:
|
|
43
|
+
pl = o.get("payload", o) if isinstance(o, dict) else {}
|
|
44
|
+
c["payload:" + str(pl.get("type"))] += 1
|
|
45
|
+
if pl.get("type") in ("function_call", "custom_tool_call"):
|
|
46
|
+
c["tool:" + str(pl.get("name"))] += 1
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
for kind, root, pat in readers.default_roots():
|
|
50
|
+
files = sorted((p for p in root.glob(pat) if p.is_file()), key=lambda p: p.stat().st_mtime)[-n:] \
|
|
51
|
+
if root.exists() else []
|
|
52
|
+
if not files:
|
|
53
|
+
print("READER", kind, str(root).replace(str(Path.home()), "~"), "files 0")
|
|
54
|
+
continue
|
|
55
|
+
ev, errs, empty, verdicts = collections.Counter(), collections.Counter(), 0, collections.Counter()
|
|
56
|
+
for p in files:
|
|
57
|
+
s = readers.read(kind, p)
|
|
58
|
+
if s.error:
|
|
59
|
+
errs[s.error.split(":")[0]] += 1
|
|
60
|
+
continue
|
|
61
|
+
try:
|
|
62
|
+
structure(kind, p)
|
|
63
|
+
except (OSError, ValueError, AttributeError) as e: # structure census only; the reader result stands
|
|
64
|
+
errs["census-" + type(e).__name__] += 1
|
|
65
|
+
k = collections.Counter(e[0] for e in s.events)
|
|
66
|
+
ev.update(k)
|
|
67
|
+
empty += not s.events
|
|
68
|
+
for layer in ("v2", "sealed"):
|
|
69
|
+
for c in scan.claims_of(s, layer):
|
|
70
|
+
verdicts[f"{layer}:{c['verdict']}"] += 1
|
|
71
|
+
print("READER", kind, str(root).replace(str(Path.home()), "~"), json.dumps(
|
|
72
|
+
{"files": len(files), "unreadable": dict(errs), "no_events": empty, "events": dict(ev),
|
|
73
|
+
"claims": dict(sorted(verdicts.items()))}))
|
|
74
|
+
for kind, c in SEEN.items():
|
|
75
|
+
handled = set(readers.EDIT_TOOLS) | set(readers.SHELL_TOOLS)
|
|
76
|
+
unhandled = {k: v for k, v in c.most_common() if k.startswith("tool:") and k[5:] not in handled}
|
|
77
|
+
print("STRUCTURE", kind, json.dumps({"records": {k: v for k, v in c.most_common() if not k.startswith("tool:")},
|
|
78
|
+
"tools_not_read_as_cmds": dict(list(unhandled.items())[:25])}))
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Screen the sealed and the corrected (v2) command-signal layers over public SWE-bench Verified trajectories and
|
|
2
|
+
print every run where their reading of the agent's own last test differs.
|
|
3
|
+
|
|
4
|
+
Usage: PYTHONPATH=src python3 scripts/screen_layers.py <data dir> <per submission> <submission> ...
|
|
5
|
+
<data dir> holds <submission>/trajs/<files> as downloaded from s3://swe-bench-submissions/verified/. From each named
|
|
6
|
+
submission, up to <per submission> trajectories are drawn round-robin across repositories (seed 20261115), so every
|
|
7
|
+
repository is represented. For each run the commands before its last success claim (or all commands when it makes
|
|
8
|
+
none) are read by both layers. Output: one SUMMARY line per submission, one DIFF block per disagreement with the
|
|
9
|
+
deciding command and its output as each layer saw it, and a TOTAL line.
|
|
10
|
+
"""
|
|
11
|
+
import json
|
|
12
|
+
import random
|
|
13
|
+
import sys
|
|
14
|
+
from collections import defaultdict
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from saiddone import readers, scan
|
|
18
|
+
from saiddone.rules import LAYERS, own_test_state, signals, signals_v2
|
|
19
|
+
|
|
20
|
+
data, per, subs = Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
|
|
21
|
+
rng = random.Random(20261115)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def pick(sub: str) -> list[Path]:
|
|
25
|
+
if not (data / sub / "trajs").is_dir():
|
|
26
|
+
sys.exit(f"REFUSED: {data / sub / 'trajs'} is not a readable directory on this host")
|
|
27
|
+
files =sorted(p for p in (data / sub / "trajs").rglob("*") if p.is_file()
|
|
28
|
+
and (p.name.endswith((".traj", ".traj.json"))))
|
|
29
|
+
by_repo = defaultdict(list)
|
|
30
|
+
for p in files:
|
|
31
|
+
by_repo[p.name.split("__")[0]].append(p)
|
|
32
|
+
for v in by_repo.values():
|
|
33
|
+
rng.shuffle(v)
|
|
34
|
+
out, repos = [], sorted(by_repo)
|
|
35
|
+
while len(out) < per and any(by_repo.values()):
|
|
36
|
+
for r in repos:
|
|
37
|
+
if by_repo[r] and len(out) < per:
|
|
38
|
+
out.append(by_repo[r].pop())
|
|
39
|
+
if not out:
|
|
40
|
+
sys.exit(f"REFUSED: no .traj or .traj.json files under {data / sub / 'trajs'}")
|
|
41
|
+
return out
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def deciding(cmds, layer):
|
|
45
|
+
lay = LAYERS[layer]
|
|
46
|
+
acts = [a for a, _ in cmds]
|
|
47
|
+
sub_i = max([i for i, a in enumerate(acts) if lay.submit_re.search(a)] or [len(acts)])
|
|
48
|
+
out = {}
|
|
49
|
+
for i in range(sub_i - 1, -1, -1):
|
|
50
|
+
if "test" not in out and lay.is_test(cmds[i][0]):
|
|
51
|
+
out["test"] = {"i": i, "cmd": cmds[i][0][:300], "out_tail": cmds[i][1][-900:]}
|
|
52
|
+
if "edit" not in out and lay.is_edit(cmds[i][0]):
|
|
53
|
+
out["edit"] = {"i": i, "cmd": cmds[i][0][:200]}
|
|
54
|
+
return out
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
tot = defaultdict(int)
|
|
58
|
+
for sub in subs:
|
|
59
|
+
n = claims = diffs = unread = 0
|
|
60
|
+
by_state = defaultdict(int)
|
|
61
|
+
for f in pick(sub):
|
|
62
|
+
s = readers.read("swe-bench", f)
|
|
63
|
+
n += 1
|
|
64
|
+
if s.error:
|
|
65
|
+
unread += 1
|
|
66
|
+
continue
|
|
67
|
+
cl = scan.claims_of(s)
|
|
68
|
+
cmds_ev = [e for e in s.events if e[0] == "cmd"]
|
|
69
|
+
k = cl[-1]["commands_before_claim"] if cl else len(cmds_ev)
|
|
70
|
+
claims += bool(cl)
|
|
71
|
+
pairs = [(a, o) for _, a, o, _ in cmds_ev[:k]]
|
|
72
|
+
a, b = signals.signals(pairs), signals_v2.signals(pairs)
|
|
73
|
+
sa, sb = own_test_state(a), own_test_state(b)
|
|
74
|
+
by_state[(sa, sb)] += 1
|
|
75
|
+
if sa != sb or a["own_test"] != b["own_test"]:
|
|
76
|
+
diffs += 1
|
|
77
|
+
print("DIFF", json.dumps({"sub": sub, "file": f.name, "claim": cl[-1]["claim_class"] if cl else None,
|
|
78
|
+
"sealed": sa, "v2": sb, "n_cmds": len(pairs),
|
|
79
|
+
"sealed_at": deciding(pairs, "sealed"), "v2_at": deciding(pairs, "v2")}),
|
|
80
|
+
flush=True)
|
|
81
|
+
print("SUMMARY", sub, json.dumps({"runs": n, "unreadable": unread, "with_claim": claims, "diffs": diffs,
|
|
82
|
+
"pairs": {f"{x}->{y}": c for (x, y), c in sorted(by_state.items())}}),
|
|
83
|
+
flush=True)
|
|
84
|
+
for key, v in (("runs", n), ("unreadable", unread), ("with_claim", claims), ("diffs", diffs)):
|
|
85
|
+
tot[key] += v
|
|
86
|
+
print("TOTAL", json.dumps(tot), flush=True)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""saiddone: when a coding agent says "done", "fixed" or "tests pass", check what its own commands showed.
|
|
2
|
+
|
|
3
|
+
saiddone demo recorded real sessions bundled with the package; needs no logs
|
|
4
|
+
saiddone scan Claude Code, Codex CLI and Gemini CLI transcripts on this machine
|
|
5
|
+
saiddone scan --swebench <dir> SWE-bench trajectories (SWE-agent .traj, mini-swe-agent .traj.json)
|
|
6
|
+
saiddone scan --path <file-or-dir> any transcript files; the format is detected per file
|
|
7
|
+
saiddone scan --json > report.json the full report as JSON
|
|
8
|
+
|
|
9
|
+
Verdicts: backed (a passing test ran after the last edit), stale green (a test passed, then the code was edited),
|
|
10
|
+
contradicted (the last test run failed), unbacked (no test ran). Offline: no account, no key, no model call.
|
|
11
|
+
Rules: the corrected layer v2 by default; --rules sealed gives the study's sealed layer byte for byte. Every verdict
|
|
12
|
+
names the layer that produced it.
|
|
13
|
+
Exit 0; with --strict, exit 1 when any claim is contradicted.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import importlib.resources
|
|
20
|
+
import json
|
|
21
|
+
import sys
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from . import __version__, readers, scan
|
|
25
|
+
from .rules import DEFAULT_LAYER, DIGESTS, LAYERS
|
|
26
|
+
|
|
27
|
+
SUFFIXES = (".jsonl", ".json", ".traj")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def detect(p: Path) -> str:
|
|
31
|
+
if p.name.endswith((".traj", ".traj.json")):
|
|
32
|
+
return "swe-bench"
|
|
33
|
+
with p.open(encoding="utf-8", errors="replace") as f:
|
|
34
|
+
head = f.read(4000)
|
|
35
|
+
if p.name.endswith(".jsonl"):
|
|
36
|
+
return "codex-cli" if '"payload"' in head or '"session_meta"' in head else "claude-code"
|
|
37
|
+
if '"projectHash"' in head or '"type": "gemini"' in head or '"type":"gemini"' in head:
|
|
38
|
+
return "gemini-cli"
|
|
39
|
+
return "swe-bench"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def files_of(paths: list[str], kind: str | None) -> list[tuple[str, Path]]:
|
|
43
|
+
out = []
|
|
44
|
+
for raw in paths:
|
|
45
|
+
p = Path(raw).expanduser()
|
|
46
|
+
found = sorted(x for x in p.rglob("*") if x.is_file() and x.name.endswith(SUFFIXES)) if p.is_dir() else [p]
|
|
47
|
+
out += [(kind or detect(f), f) for f in found if f.is_file()]
|
|
48
|
+
return out
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def scan_files(files: list[tuple[str, Path]], searched: list[dict], layer: str = DEFAULT_LAYER) -> dict:
|
|
52
|
+
rep = scan.run([], None, layer)
|
|
53
|
+
rep["searched"] = searched
|
|
54
|
+
for kind, f in files:
|
|
55
|
+
s = readers.read(kind, f)
|
|
56
|
+
rep["sessions_found"] += 1
|
|
57
|
+
if s.error:
|
|
58
|
+
rep["sessions_unreadable"].append({"file": str(f), "error": s.error})
|
|
59
|
+
continue
|
|
60
|
+
rep["sessions_parsed"] += 1
|
|
61
|
+
rep["claims"] += scan.claims_of(s, layer)
|
|
62
|
+
rep["claims_found"] = len(rep["claims"])
|
|
63
|
+
rep["verdicts"] = {v: sum(c["verdict"] == v for c in rep["claims"]) for v in rep["verdicts"]}
|
|
64
|
+
return rep
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def output(rep: dict, a) -> int:
|
|
68
|
+
if a.json:
|
|
69
|
+
print(json.dumps(rep, indent=1))
|
|
70
|
+
elif rep["sessions_found"] == 0:
|
|
71
|
+
print(scan.render_empty(rep))
|
|
72
|
+
else:
|
|
73
|
+
print(scan.render(rep, None if a.all else 20))
|
|
74
|
+
return 1 if a.strict and rep["verdicts"]["contradicted"] else 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def demo(a) -> int:
|
|
78
|
+
d = Path(str(importlib.resources.files("saiddone").joinpath("data/demo")))
|
|
79
|
+
man = json.loads((d / "MANIFEST.json").read_text(encoding="utf-8"))
|
|
80
|
+
files = [(x["reader"], d / x["file"]) for x in man["sessions"]]
|
|
81
|
+
rep = scan_files(files, [{"reader": "demo", "path": "saiddone/data/demo", "pattern": "MANIFEST.json",
|
|
82
|
+
"exists": True, "files": len(files)}], a.rules)
|
|
83
|
+
for c in rep["claims"]:
|
|
84
|
+
c["file"] = "demo/" + Path(c["file"]).name
|
|
85
|
+
for ev in c["evidence"].values():
|
|
86
|
+
ev.setdefault("line", None)
|
|
87
|
+
rep["demo_sources"] = man["sessions"]
|
|
88
|
+
if not a.json:
|
|
89
|
+
print("Recorded real sessions bundled with saiddone. Where each came from: saiddone/data/demo/MANIFEST.json\n")
|
|
90
|
+
a.all = True
|
|
91
|
+
return output(rep, a)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def main(argv: list[str] | None = None) -> int:
|
|
95
|
+
p = argparse.ArgumentParser(prog="saiddone", description=__doc__,
|
|
96
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
97
|
+
p.add_argument("--version", action="version", version=f"saiddone {__version__}; rules "
|
|
98
|
+
+ ", ".join(f"{k} {v}" for k, v in DIGESTS.items()))
|
|
99
|
+
sp = p.add_subparsers(dest="cmd", required=True)
|
|
100
|
+
for name in ("demo", "scan"):
|
|
101
|
+
q = sp.add_parser(name)
|
|
102
|
+
q.add_argument("--json", action="store_true", help="print the full report as JSON")
|
|
103
|
+
q.add_argument("--strict", action="store_true", help="exit 1 when any claim is contradicted")
|
|
104
|
+
q.add_argument("--all", action="store_true", help="print every claim, not only the last 20")
|
|
105
|
+
q.add_argument("--rules", choices=sorted(LAYERS), default=DEFAULT_LAYER,
|
|
106
|
+
help="rule layer: v2 (corrected, default) or sealed (the study's sealed layer, byte for byte)")
|
|
107
|
+
if name == "scan":
|
|
108
|
+
q.add_argument("--swebench", action="append", default=[], metavar="DIR")
|
|
109
|
+
q.add_argument("--path", action="append", default=[], metavar="FILE_OR_DIR")
|
|
110
|
+
q.add_argument("--since", type=float, metavar="DAYS", help="only sessions changed in the last DAYS days")
|
|
111
|
+
a = p.parse_args(argv)
|
|
112
|
+
if a.cmd == "demo":
|
|
113
|
+
return demo(a)
|
|
114
|
+
if a.path or a.swebench:
|
|
115
|
+
files = files_of(a.path, None) + files_of(a.swebench, "swe-bench")
|
|
116
|
+
searched = [{"reader": "swe-bench" if x in a.swebench else "detected per file", "path": x, "pattern": "",
|
|
117
|
+
"exists": Path(x).expanduser().exists(), "files": len(files_of([x], None))}
|
|
118
|
+
for x in a.path + a.swebench]
|
|
119
|
+
return output(scan_files(files, searched, a.rules), a)
|
|
120
|
+
return output(scan.run(readers.default_roots(), a.since, a.rules), a)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
if __name__ == "__main__":
|
|
124
|
+
sys.exit(main())
|