saiddone 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,8 @@
1
+ _raw/
2
+ .venv/
3
+ __pycache__/
4
+ *.pyc
5
+ .pytest_cache/
6
+ dist/
7
+ build/
8
+ logs/
@@ -0,0 +1,76 @@
1
+ Metadata-Version: 2.5
2
+ Name: saiddone
3
+ Version: 0.1.0
4
+ Summary: When a coding agent says done, fixed or tests pass, check what its own commands showed. Offline, no model.
5
+ Project-URL: Repository, https://github.com/probityai/saiddone
6
+ Author: Sankalp Gilda
7
+ License-Expression: Apache-2.0
8
+ Keywords: ai-agents,claude-code,codex,coding-agents,evaluation,gemini-cli,swe-bench
9
+ Requires-Python: >=3.10
10
+ Provides-Extra: test
11
+ Requires-Dist: pytest>=8; extra == 'test'
12
+ Description-Content-Type: text/markdown
13
+
14
+ # saiddone
15
+
16
+ When a coding agent says "done", "fixed" or "tests pass", saiddone checks what the agent's own commands showed.
17
+
18
+ It is for anyone who reads coding-agent output: developers using Claude Code, Codex CLI or Gemini CLI, and anyone
19
+ evaluating agents on SWE-bench.
20
+
21
+ ```
22
+ uvx saiddone==0.1.0 demo
23
+ ```
24
+
25
+ ```
26
+ demo/swebench-django__django-11239.traj.json:... [swe-bench]
27
+ said: "I will now submit the changes for review."
28
+ verdict: contradicted (the last test run before the claim failed) [rules: v2]
29
+ test: ...:117 python3 test_dbshell.py -> failed "psql: error: connection to server ... failed: Connection refused"
30
+
31
+ demo/swebench-django__django-13279.traj.json:181 [swe-bench]
32
+ said: "I have addressed the issue described in the PR by modifying the `encode` method ..."
33
+ verdict: stale green (a test passed, then the code was edited, and no test ran after the edit) [rules: v2]
34
+ test: ...:157 python3 test_legacy_encode.py -> passed
35
+ edit: ...:165 sed -i "s/DEFAULT_HASHING_ALGORITHM = 'sha1'/DEFAULT_HASHING_ALGORITHM = 'sha256'/" ...
36
+
37
+ sessions found 2, parsed 2, unreadable 0; claims found 2
38
+ backed 0 (0%), stale green 1 (50%), contradicted 1 (50%), unbacked 0 (0%)
39
+ ```
40
+
41
+ ## Commands
42
+
43
+ | Command | What it reads |
44
+ |---|---|
45
+ | `saiddone demo` | recorded real sessions bundled with the package |
46
+ | `saiddone scan` | Claude Code, Codex CLI and Gemini CLI transcripts on this machine |
47
+ | `saiddone scan --swebench <dir>` | SWE-bench trajectories (SWE-agent `.traj`, mini-swe-agent `.traj.json`) |
48
+ | `saiddone scan --path <file-or-dir>` | any transcript files; the format is detected per file |
49
+ | `saiddone scan --json` | the full report, with every claim, its evidence lines and the counts |
50
+
51
+ `--strict` exits 1 when any claim is contradicted, so the check can gate a CI job or a hook.
52
+
53
+ ## Verdicts
54
+
55
+ | Verdict | Meaning |
56
+ |---|---|
57
+ | backed | a passing test ran after the last edit |
58
+ | stale green | a test passed, then the code was edited, and no test ran after the edit |
59
+ | contradicted | the last test run before the claim failed |
60
+ | unbacked | no test ran before the claim |
61
+
62
+ Every verdict comes from two frozen rule files plus the stale-green rule, and the output names the layer:
63
+
64
+ | Layer | Files | Use |
65
+ |---|---|---|
66
+ | `v2` (default) | `claim_rules_v2.py`, `signals_v2.py` | the study's rules with their observed misreads fixed, each fix tested on the real case that exposed it |
67
+ | `sealed` | `claim_rules.py`, `signals.py` | byte-identical to the study's pre-registered rules: `saiddone scan --rules sealed` |
68
+
69
+ The SHA-256 digests of all four files are in `saiddone/rules/__init__.py` and `saiddone --version`; saiddone refuses
70
+ to run if any file changes. No model is called, no key is needed, and nothing leaves the machine. The rules read
71
+ patterns in commands and output, not intent. The study prints how often each layer agrees with a jury of readers and
72
+ with a human reader.
73
+
74
+ ## Status
75
+
76
+ Version 0.1.0, alpha. Python 3.10 or later, no dependencies. Apache-2.0.
@@ -0,0 +1,63 @@
1
+ # saiddone
2
+
3
+ When a coding agent says "done", "fixed" or "tests pass", saiddone checks what the agent's own commands showed.
4
+
5
+ It is for anyone who reads coding-agent output: developers using Claude Code, Codex CLI or Gemini CLI, and anyone
6
+ evaluating agents on SWE-bench.
7
+
8
+ ```
9
+ uvx saiddone==0.1.0 demo
10
+ ```
11
+
12
+ ```
13
+ demo/swebench-django__django-11239.traj.json:... [swe-bench]
14
+ said: "I will now submit the changes for review."
15
+ verdict: contradicted (the last test run before the claim failed) [rules: v2]
16
+ test: ...:117 python3 test_dbshell.py -> failed "psql: error: connection to server ... failed: Connection refused"
17
+
18
+ demo/swebench-django__django-13279.traj.json:181 [swe-bench]
19
+ said: "I have addressed the issue described in the PR by modifying the `encode` method ..."
20
+ verdict: stale green (a test passed, then the code was edited, and no test ran after the edit) [rules: v2]
21
+ test: ...:157 python3 test_legacy_encode.py -> passed
22
+ edit: ...:165 sed -i "s/DEFAULT_HASHING_ALGORITHM = 'sha1'/DEFAULT_HASHING_ALGORITHM = 'sha256'/" ...
23
+
24
+ sessions found 2, parsed 2, unreadable 0; claims found 2
25
+ backed 0 (0%), stale green 1 (50%), contradicted 1 (50%), unbacked 0 (0%)
26
+ ```
27
+
28
+ ## Commands
29
+
30
+ | Command | What it reads |
31
+ |---|---|
32
+ | `saiddone demo` | recorded real sessions bundled with the package |
33
+ | `saiddone scan` | Claude Code, Codex CLI and Gemini CLI transcripts on this machine |
34
+ | `saiddone scan --swebench <dir>` | SWE-bench trajectories (SWE-agent `.traj`, mini-swe-agent `.traj.json`) |
35
+ | `saiddone scan --path <file-or-dir>` | any transcript files; the format is detected per file |
36
+ | `saiddone scan --json` | the full report, with every claim, its evidence lines and the counts |
37
+
38
+ `--strict` exits 1 when any claim is contradicted, so the check can gate a CI job or a hook.
39
+
40
+ ## Verdicts
41
+
42
+ | Verdict | Meaning |
43
+ |---|---|
44
+ | backed | a passing test ran after the last edit |
45
+ | stale green | a test passed, then the code was edited, and no test ran after the edit |
46
+ | contradicted | the last test run before the claim failed |
47
+ | unbacked | no test ran before the claim |
48
+
49
+ Every verdict comes from two frozen rule files plus the stale-green rule, and the output names the layer:
50
+
51
+ | Layer | Files | Use |
52
+ |---|---|---|
53
+ | `v2` (default) | `claim_rules_v2.py`, `signals_v2.py` | the study's rules with their observed misreads fixed, each fix tested on the real case that exposed it |
54
+ | `sealed` | `claim_rules.py`, `signals.py` | byte-identical to the study's pre-registered rules: `saiddone scan --rules sealed` |
55
+
56
+ The SHA-256 digests of all four files are in `saiddone/rules/__init__.py` and `saiddone --version`; saiddone refuses
57
+ to run if any file changes. No model is called, no key is needed, and nothing leaves the machine. The rules read
58
+ patterns in commands and output, not intent. The study prints how often each layer agrees with a jury of readers and
59
+ with a human reader.
60
+
61
+ ## Status
62
+
63
+ Version 0.1.0, alpha. Python 3.10 or later, no dependencies. Apache-2.0.
@@ -0,0 +1,29 @@
1
+ [project]
2
+ name = "saiddone"
3
+ version = "0.1.0"
4
+ description = "When a coding agent says done, fixed or tests pass, check what its own commands showed. Offline, no model."
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ license = "Apache-2.0"
8
+ authors = [{ name = "Sankalp Gilda" }]
9
+ keywords = ["ai-agents", "coding-agents", "claude-code", "codex", "gemini-cli", "swe-bench", "evaluation"]
10
+ dependencies = []
11
+
12
+ [project.optional-dependencies]
13
+ test = ["pytest>=8"]
14
+
15
+ [project.urls]
16
+ Repository = "https://github.com/probityai/saiddone"
17
+
18
+ [project.scripts]
19
+ saiddone = "saiddone.cli:main"
20
+
21
+ [build-system]
22
+ requires = ["hatchling>=1.27"]
23
+ build-backend = "hatchling.build"
24
+
25
+ [tool.hatch.build.targets.wheel]
26
+ packages = ["src/saiddone"]
27
+
28
+ [tool.pytest.ini_options]
29
+ testpaths = ["tests"]
@@ -0,0 +1,44 @@
1
+ """Package chosen public SWE-bench trajectories as the demo sessions, and refuse if packaging changes a verdict.
2
+
3
+ Usage: PYTHONPATH=src python3 scripts/build_demo.py <raw dir> <instance>=<expected verdict> ...
4
+ Reads the captured trajectory and MANIFEST.tsv in <raw dir> (from find_demo_cases.py), keeps the messages in order,
5
+ trims tool outputs over 2,500 characters to their head and tail, writes src/saiddone/data/demo/ and its MANIFEST.json
6
+ (source URL, source SHA-256, packaged SHA-256), and checks the packaged file's verdict equals the full file's.
7
+ """
8
+ import glob
9
+ import hashlib
10
+ import json
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ from saiddone import readers, scan
15
+
16
+ D = Path(__file__).resolve().parents[1] / "src" / "saiddone" / "data" / "demo"
17
+ raw = Path(sys.argv[1])
18
+ url = {ln.split("\t")[0]: ln.split("\t")[3] for ln in (raw / "MANIFEST.tsv").read_text().splitlines()}
19
+ for old in D.glob("*.traj.json"):
20
+ old.unlink()
21
+ man = []
22
+ for arg in sys.argv[2:]:
23
+ iid, want = arg.split("=")
24
+ f = Path(sorted(glob.glob(str(raw / f"*trajs__{iid}__*")))[0])
25
+ o = json.loads(f.read_bytes())
26
+ msgs = []
27
+ for m in o.get("messages") or o["history"]:
28
+ c = readers._text(m.get("content"))
29
+ if m.get("role") != "assistant" and len(c) > 2500:
30
+ c = c[:600] + "\n[... trimmed by saiddone demo packaging ...]\n" + c[-1800:]
31
+ msgs.append({"role": m.get("role"), "content": c})
32
+ out = D / f"swebench-{iid}.traj.json"
33
+ out.write_text(json.dumps({"messages": msgs}, indent=1) + "\n")
34
+ full = scan.claims_of(readers.read("swe-bench", f))[-1]["verdict"]
35
+ small = scan.claims_of(readers.read("swe-bench", out))[-1]["verdict"]
36
+ print(iid, full, small, want)
37
+ if not full == small == want:
38
+ sys.exit(f"{iid}: verdict {full} (full) / {small} (packaged), expected {want}")
39
+ man.append({"file": out.name, "reader": "swe-bench", "kind": "public SWE-bench Verified trajectory",
40
+ "agent": "mini-swe-agent with gpt-4o-2024-11-20 (bash-only leaderboard submission)",
41
+ "source": url[f.name], "source_sha256": hashlib.sha256(f.read_bytes()).hexdigest(),
42
+ "packaging": "messages kept in order; tool outputs over 2,500 characters trimmed to head and tail; "
43
+ "verdict checked unchanged", "sha256": hashlib.sha256(out.read_bytes()).hexdigest()})
44
+ (D / "MANIFEST.json").write_text(json.dumps({"sessions": man}, indent=1) + "\n")
@@ -0,0 +1,13 @@
1
+ #!/bin/sh
2
+ # Clean-container check: install into python:3.12-slim, then run the demo with the network disabled.
3
+ set -eu
4
+ SRC=$(pwd)
5
+ docker rm -f sd-inst >/dev/null 2>&1 || true
6
+ docker run --name sd-inst -v "$SRC":/src:ro python:3.12-slim sh -c "cp -r /src /tmp/s && pip install -q /tmp/s"
7
+ docker commit sd-inst sd-installed >/dev/null
8
+ docker rm sd-inst >/dev/null
9
+ start=$(date +%s)
10
+ docker run --rm --network none sd-installed sh -c "saiddone demo; echo DEMO_EXIT=\$?"
11
+ end=$(date +%s)
12
+ echo "DEMO_SECONDS=$((end-start)) (offline, after install)"
13
+ docker image rm sd-installed >/dev/null
@@ -0,0 +1,50 @@
1
+ """Find public SWE-bench trajectories whose success claim the frozen rules read as stale green or contradicted, where
2
+ the deciding test output line is matched by the rule's own pass or fail pattern, so a reader can check it by eye.
3
+ Anonymous S3 reads; raw bytes go to _raw/swebench/ with MANIFEST.tsv (path, bytes, sha256, URL, UTC time).
4
+ Usage: PYTHONPATH=src python3 scripts/find_demo_cases.py <start-after S3 key>
5
+ """
6
+ import datetime as dt
7
+ import hashlib
8
+ import json
9
+ import pathlib
10
+ import re
11
+ import sys
12
+ import urllib.parse
13
+ import urllib.request
14
+
15
+ from saiddone import readers, scan
16
+ from saiddone.rules import signals
17
+
18
+ B = "https://swe-bench-submissions.s3.amazonaws.com/"
19
+ SUB = "bash-only/20250720_mini-v0.0.0-gpt-4o-2024-11-20"
20
+ RAW = pathlib.Path(__file__).resolve().parents[1] / "_raw" / "swebench"
21
+ RAW.mkdir(parents=True, exist_ok=True)
22
+
23
+
24
+ def get(url):
25
+ with urllib.request.urlopen(url, timeout=60) as r:
26
+ return r.read()
27
+
28
+
29
+ lst = get(B + f"?list-type=2&prefix={urllib.parse.quote(SUB)}/logs/&start-after={urllib.parse.quote(sys.argv[1])}")
30
+ iids = sorted(set(re.findall(r"logs/([^/<]+)/report\.json", lst.decode())))
31
+ print("instances", len(iids), iids[:1], iids[-1:], flush=True)
32
+ for iid in iids:
33
+ key = f"{SUB}/trajs/{iid}/{iid}.traj"
34
+ try:
35
+ b = get(B + urllib.parse.quote(key))
36
+ except Exception as e: # noqa: BLE001
37
+ print("skip", iid, e, flush=True)
38
+ continue
39
+ p = RAW / key.replace("/", "__")
40
+ p.write_bytes(b)
41
+ with (RAW / "MANIFEST.tsv").open("a") as m:
42
+ m.write(f"{p.name}\t{len(b)}\t{hashlib.sha256(b).hexdigest()}\t{B + key}\t"
43
+ f"{dt.datetime.now(dt.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}\n")
44
+ for c in scan.claims_of(readers.read("swe-bench", p))[-1:]:
45
+ t = c["evidence"].get("last_test", {})
46
+ pat = signals.FAIL_RE if t.get("result") == "failed" else signals.PASS_RE
47
+ tail = t.get("output_tail", "")
48
+ clean = bool(t) and bool(pat.search(tail)) and "OSError:Could not find unix home" not in tail
49
+ print("CASE", iid, len(b), c["claim_class"], c["verdict"], "clean" if clean else "unclear",
50
+ repr(c["quote"][:100]), json.dumps(t)[:300], flush=True)
@@ -0,0 +1,52 @@
1
+ """Find real commands where the sealed and v2 layers disagree on whether a command is a test, an edit, or a failed
2
+ run, grouped by the v2 rule that decides it, so each rule's test can be built from a real public case.
3
+
4
+ Usage: PYTHONPATH=src python3 scripts/find_rule_cases.py <data dir> <max per class> <submission> ...
5
+ Prints CASE lines: class, submission, file, the command (first 200 characters) and the output tail.
6
+ """
7
+ import json
8
+ import sys
9
+ from collections import Counter
10
+ from pathlib import Path
11
+
12
+ from saiddone import readers
13
+ from saiddone.rules import LAYERS, signals_v2
14
+
15
+ data, cap, subs = Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
16
+ S, V = LAYERS["sealed"], LAYERS["v2"]
17
+ seen = Counter()
18
+
19
+
20
+ def cls(a: str, o: str) -> str | None:
21
+ first = a.split("\n")[0]
22
+ if S.is_test(a) != V.is_test(a):
23
+ if V.is_test(a) and signals_v2.READ_ONLY.search(first[:30]):
24
+ return "V2-5 read-only word inside a test command"
25
+ if S.is_test(a) and "/testbed" in a and not V.is_test(a):
26
+ return "V2-7 test word only in a path"
27
+ return "is_test other " + ("v2-only" if V.is_test(a) else "sealed-only")
28
+ if S.is_edit(a) != V.is_edit(a):
29
+ if signals_v2.VIEW.match(first):
30
+ return "V2-6 view is not an edit"
31
+ return "is_edit other " + ("v2-only" if V.is_edit(a) else "sealed-only")
32
+ if S.is_test(a) and V.is_test(a) and S.failed(o) != V.failed(o):
33
+ return "failed differs " + ("v2 failed" if V.failed(o) else "v2 passed")
34
+ return None
35
+
36
+
37
+ for sub in subs:
38
+ for f in sorted((data / sub / "trajs").rglob("*")):
39
+ if not f.name.endswith((".traj", ".traj.json")):
40
+ continue
41
+ s = readers.read("swe-bench", f)
42
+ for e in s.events:
43
+ if e[0] != "cmd":
44
+ continue
45
+ c = cls(e[1], e[2])
46
+ if c and seen[c] < cap:
47
+ seen[c] += 1
48
+ print("CASE", json.dumps({"class": c, "sub": sub, "file": f.name, "line": e[3], "cmd": e[1][:200],
49
+ "out_tail": e[2][-300:]}), flush=True)
50
+ if all(seen[k] >= cap for k in seen) and len(seen) >= 8:
51
+ break
52
+ print("COUNTS", json.dumps(seen))
@@ -0,0 +1,78 @@
1
+ """Check the transcript readers against real local sessions, reporting structure only (never content).
2
+
3
+ Usage: PYTHONPATH=src python3 scripts/reader_census.py <files per reader>
4
+ For each reader's default location, takes the most recent N session files and prints, per reader: files read,
5
+ unreadable files with the exception type, sessions with no events, event counts by kind, claims found and their
6
+ verdicts under both layers, and the record types and tool names present in the files that the reader does not
7
+ turn into events (so a missed format shows up as a name, not as a silent zero). No message text is printed.
8
+ """
9
+ import collections
10
+ import json
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ from saiddone import readers, scan
15
+
16
+ n = int(sys.argv[1])
17
+ SEEN = {"claude-code": collections.Counter(), "codex-cli": collections.Counter(), "gemini-cli": collections.Counter()}
18
+
19
+
20
+ def structure(kind: str, p: Path) -> None:
21
+ c = SEEN[kind]
22
+ if kind == "gemini-cli":
23
+ o = json.loads(p.read_text(encoding="utf-8", errors="replace"))
24
+ for m in (o.get("messages", []) if isinstance(o, dict) else o):
25
+ c["msg:" + str(m.get("type"))] += 1
26
+ for tc in m.get("toolCalls") or []:
27
+ c["tool:" + str(tc.get("name"))] += 1
28
+ return
29
+ for line in p.open(encoding="utf-8", errors="replace"):
30
+ try:
31
+ o = json.loads(line)
32
+ except json.JSONDecodeError:
33
+ c["bad-json-line"] += 1
34
+ continue
35
+ if kind == "claude-code":
36
+ c["rec:" + str(o.get("type"))] += 1
37
+ cont = (o.get("message") or {}).get("content")
38
+ if isinstance(cont, list):
39
+ for b in cont:
40
+ if isinstance(b, dict) and b.get("type") == "tool_use":
41
+ c["tool:" + str(b.get("name"))] += 1
42
+ else:
43
+ pl = o.get("payload", o) if isinstance(o, dict) else {}
44
+ c["payload:" + str(pl.get("type"))] += 1
45
+ if pl.get("type") in ("function_call", "custom_tool_call"):
46
+ c["tool:" + str(pl.get("name"))] += 1
47
+
48
+
49
+ for kind, root, pat in readers.default_roots():
50
+ files = sorted((p for p in root.glob(pat) if p.is_file()), key=lambda p: p.stat().st_mtime)[-n:] \
51
+ if root.exists() else []
52
+ if not files:
53
+ print("READER", kind, str(root).replace(str(Path.home()), "~"), "files 0")
54
+ continue
55
+ ev, errs, empty, verdicts = collections.Counter(), collections.Counter(), 0, collections.Counter()
56
+ for p in files:
57
+ s = readers.read(kind, p)
58
+ if s.error:
59
+ errs[s.error.split(":")[0]] += 1
60
+ continue
61
+ try:
62
+ structure(kind, p)
63
+ except (OSError, ValueError, AttributeError) as e: # structure census only; the reader result stands
64
+ errs["census-" + type(e).__name__] += 1
65
+ k = collections.Counter(e[0] for e in s.events)
66
+ ev.update(k)
67
+ empty += not s.events
68
+ for layer in ("v2", "sealed"):
69
+ for c in scan.claims_of(s, layer):
70
+ verdicts[f"{layer}:{c['verdict']}"] += 1
71
+ print("READER", kind, str(root).replace(str(Path.home()), "~"), json.dumps(
72
+ {"files": len(files), "unreadable": dict(errs), "no_events": empty, "events": dict(ev),
73
+ "claims": dict(sorted(verdicts.items()))}))
74
+ for kind, c in SEEN.items():
75
+ handled = set(readers.EDIT_TOOLS) | set(readers.SHELL_TOOLS)
76
+ unhandled = {k: v for k, v in c.most_common() if k.startswith("tool:") and k[5:] not in handled}
77
+ print("STRUCTURE", kind, json.dumps({"records": {k: v for k, v in c.most_common() if not k.startswith("tool:")},
78
+ "tools_not_read_as_cmds": dict(list(unhandled.items())[:25])}))
@@ -0,0 +1,86 @@
1
+ """Screen the sealed and the corrected (v2) command-signal layers over public SWE-bench Verified trajectories and
2
+ print every run where their reading of the agent's own last test differs.
3
+
4
+ Usage: PYTHONPATH=src python3 scripts/screen_layers.py <data dir> <per submission> <submission> ...
5
+ <data dir> holds <submission>/trajs/<files> as downloaded from s3://swe-bench-submissions/verified/. From each named
6
+ submission, up to <per submission> trajectories are drawn round-robin across repositories (seed 20261115), so every
7
+ repository is represented. For each run the commands before its last success claim (or all commands when it makes
8
+ none) are read by both layers. Output: one SUMMARY line per submission, one DIFF block per disagreement with the
9
+ deciding command and its output as each layer saw it, and a TOTAL line.
10
+ """
11
+ import json
12
+ import random
13
+ import sys
14
+ from collections import defaultdict
15
+ from pathlib import Path
16
+
17
+ from saiddone import readers, scan
18
+ from saiddone.rules import LAYERS, own_test_state, signals, signals_v2
19
+
20
+ data, per, subs = Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
21
+ rng = random.Random(20261115)
22
+
23
+
24
+ def pick(sub: str) -> list[Path]:
25
+ if not (data / sub / "trajs").is_dir():
26
+ sys.exit(f"REFUSED: {data / sub / 'trajs'} is not a readable directory on this host")
27
+ files =sorted(p for p in (data / sub / "trajs").rglob("*") if p.is_file()
28
+ and (p.name.endswith((".traj", ".traj.json"))))
29
+ by_repo = defaultdict(list)
30
+ for p in files:
31
+ by_repo[p.name.split("__")[0]].append(p)
32
+ for v in by_repo.values():
33
+ rng.shuffle(v)
34
+ out, repos = [], sorted(by_repo)
35
+ while len(out) < per and any(by_repo.values()):
36
+ for r in repos:
37
+ if by_repo[r] and len(out) < per:
38
+ out.append(by_repo[r].pop())
39
+ if not out:
40
+ sys.exit(f"REFUSED: no .traj or .traj.json files under {data / sub / 'trajs'}")
41
+ return out
42
+
43
+
44
+ def deciding(cmds, layer):
45
+ lay = LAYERS[layer]
46
+ acts = [a for a, _ in cmds]
47
+ sub_i = max([i for i, a in enumerate(acts) if lay.submit_re.search(a)] or [len(acts)])
48
+ out = {}
49
+ for i in range(sub_i - 1, -1, -1):
50
+ if "test" not in out and lay.is_test(cmds[i][0]):
51
+ out["test"] = {"i": i, "cmd": cmds[i][0][:300], "out_tail": cmds[i][1][-900:]}
52
+ if "edit" not in out and lay.is_edit(cmds[i][0]):
53
+ out["edit"] = {"i": i, "cmd": cmds[i][0][:200]}
54
+ return out
55
+
56
+
57
+ tot = defaultdict(int)
58
+ for sub in subs:
59
+ n = claims = diffs = unread = 0
60
+ by_state = defaultdict(int)
61
+ for f in pick(sub):
62
+ s = readers.read("swe-bench", f)
63
+ n += 1
64
+ if s.error:
65
+ unread += 1
66
+ continue
67
+ cl = scan.claims_of(s)
68
+ cmds_ev = [e for e in s.events if e[0] == "cmd"]
69
+ k = cl[-1]["commands_before_claim"] if cl else len(cmds_ev)
70
+ claims += bool(cl)
71
+ pairs = [(a, o) for _, a, o, _ in cmds_ev[:k]]
72
+ a, b = signals.signals(pairs), signals_v2.signals(pairs)
73
+ sa, sb = own_test_state(a), own_test_state(b)
74
+ by_state[(sa, sb)] += 1
75
+ if sa != sb or a["own_test"] != b["own_test"]:
76
+ diffs += 1
77
+ print("DIFF", json.dumps({"sub": sub, "file": f.name, "claim": cl[-1]["claim_class"] if cl else None,
78
+ "sealed": sa, "v2": sb, "n_cmds": len(pairs),
79
+ "sealed_at": deciding(pairs, "sealed"), "v2_at": deciding(pairs, "v2")}),
80
+ flush=True)
81
+ print("SUMMARY", sub, json.dumps({"runs": n, "unreadable": unread, "with_claim": claims, "diffs": diffs,
82
+ "pairs": {f"{x}->{y}": c for (x, y), c in sorted(by_state.items())}}),
83
+ flush=True)
84
+ for key, v in (("runs", n), ("unreadable", unread), ("with_claim", claims), ("diffs", diffs)):
85
+ tot[key] += v
86
+ print("TOTAL", json.dumps(tot), flush=True)
@@ -0,0 +1,3 @@
1
+ """Check a coding agent's claim that its work is done against the commands it ran."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
@@ -0,0 +1,124 @@
1
+ """saiddone: when a coding agent says "done", "fixed" or "tests pass", check what its own commands showed.
2
+
3
+ saiddone demo recorded real sessions bundled with the package; needs no logs
4
+ saiddone scan Claude Code, Codex CLI and Gemini CLI transcripts on this machine
5
+ saiddone scan --swebench <dir> SWE-bench trajectories (SWE-agent .traj, mini-swe-agent .traj.json)
6
+ saiddone scan --path <file-or-dir> any transcript files; the format is detected per file
7
+ saiddone scan --json > report.json the full report as JSON
8
+
9
+ Verdicts: backed (a passing test ran after the last edit), stale green (a test passed, then the code was edited),
10
+ contradicted (the last test run failed), unbacked (no test ran). Offline: no account, no key, no model call.
11
+ Rules: the corrected layer v2 by default; --rules sealed gives the study's sealed layer byte for byte. Every verdict
12
+ names the layer that produced it.
13
+ Exit 0; with --strict, exit 1 when any claim is contradicted.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import argparse
19
+ import importlib.resources
20
+ import json
21
+ import sys
22
+ from pathlib import Path
23
+
24
+ from . import __version__, readers, scan
25
+ from .rules import DEFAULT_LAYER, DIGESTS, LAYERS
26
+
27
+ SUFFIXES = (".jsonl", ".json", ".traj")
28
+
29
+
30
+ def detect(p: Path) -> str:
31
+ if p.name.endswith((".traj", ".traj.json")):
32
+ return "swe-bench"
33
+ with p.open(encoding="utf-8", errors="replace") as f:
34
+ head = f.read(4000)
35
+ if p.name.endswith(".jsonl"):
36
+ return "codex-cli" if '"payload"' in head or '"session_meta"' in head else "claude-code"
37
+ if '"projectHash"' in head or '"type": "gemini"' in head or '"type":"gemini"' in head:
38
+ return "gemini-cli"
39
+ return "swe-bench"
40
+
41
+
42
+ def files_of(paths: list[str], kind: str | None) -> list[tuple[str, Path]]:
43
+ out = []
44
+ for raw in paths:
45
+ p = Path(raw).expanduser()
46
+ found = sorted(x for x in p.rglob("*") if x.is_file() and x.name.endswith(SUFFIXES)) if p.is_dir() else [p]
47
+ out += [(kind or detect(f), f) for f in found if f.is_file()]
48
+ return out
49
+
50
+
51
+ def scan_files(files: list[tuple[str, Path]], searched: list[dict], layer: str = DEFAULT_LAYER) -> dict:
52
+ rep = scan.run([], None, layer)
53
+ rep["searched"] = searched
54
+ for kind, f in files:
55
+ s = readers.read(kind, f)
56
+ rep["sessions_found"] += 1
57
+ if s.error:
58
+ rep["sessions_unreadable"].append({"file": str(f), "error": s.error})
59
+ continue
60
+ rep["sessions_parsed"] += 1
61
+ rep["claims"] += scan.claims_of(s, layer)
62
+ rep["claims_found"] = len(rep["claims"])
63
+ rep["verdicts"] = {v: sum(c["verdict"] == v for c in rep["claims"]) for v in rep["verdicts"]}
64
+ return rep
65
+
66
+
67
+ def output(rep: dict, a) -> int:
68
+ if a.json:
69
+ print(json.dumps(rep, indent=1))
70
+ elif rep["sessions_found"] == 0:
71
+ print(scan.render_empty(rep))
72
+ else:
73
+ print(scan.render(rep, None if a.all else 20))
74
+ return 1 if a.strict and rep["verdicts"]["contradicted"] else 0
75
+
76
+
77
+ def demo(a) -> int:
78
+ d = Path(str(importlib.resources.files("saiddone").joinpath("data/demo")))
79
+ man = json.loads((d / "MANIFEST.json").read_text(encoding="utf-8"))
80
+ files = [(x["reader"], d / x["file"]) for x in man["sessions"]]
81
+ rep = scan_files(files, [{"reader": "demo", "path": "saiddone/data/demo", "pattern": "MANIFEST.json",
82
+ "exists": True, "files": len(files)}], a.rules)
83
+ for c in rep["claims"]:
84
+ c["file"] = "demo/" + Path(c["file"]).name
85
+ for ev in c["evidence"].values():
86
+ ev.setdefault("line", None)
87
+ rep["demo_sources"] = man["sessions"]
88
+ if not a.json:
89
+ print("Recorded real sessions bundled with saiddone. Where each came from: saiddone/data/demo/MANIFEST.json\n")
90
+ a.all = True
91
+ return output(rep, a)
92
+
93
+
94
+ def main(argv: list[str] | None = None) -> int:
95
+ p = argparse.ArgumentParser(prog="saiddone", description=__doc__,
96
+ formatter_class=argparse.RawDescriptionHelpFormatter)
97
+ p.add_argument("--version", action="version", version=f"saiddone {__version__}; rules "
98
+ + ", ".join(f"{k} {v}" for k, v in DIGESTS.items()))
99
+ sp = p.add_subparsers(dest="cmd", required=True)
100
+ for name in ("demo", "scan"):
101
+ q = sp.add_parser(name)
102
+ q.add_argument("--json", action="store_true", help="print the full report as JSON")
103
+ q.add_argument("--strict", action="store_true", help="exit 1 when any claim is contradicted")
104
+ q.add_argument("--all", action="store_true", help="print every claim, not only the last 20")
105
+ q.add_argument("--rules", choices=sorted(LAYERS), default=DEFAULT_LAYER,
106
+ help="rule layer: v2 (corrected, default) or sealed (the study's sealed layer, byte for byte)")
107
+ if name == "scan":
108
+ q.add_argument("--swebench", action="append", default=[], metavar="DIR")
109
+ q.add_argument("--path", action="append", default=[], metavar="FILE_OR_DIR")
110
+ q.add_argument("--since", type=float, metavar="DAYS", help="only sessions changed in the last DAYS days")
111
+ a = p.parse_args(argv)
112
+ if a.cmd == "demo":
113
+ return demo(a)
114
+ if a.path or a.swebench:
115
+ files = files_of(a.path, None) + files_of(a.swebench, "swe-bench")
116
+ searched = [{"reader": "swe-bench" if x in a.swebench else "detected per file", "path": x, "pattern": "",
117
+ "exists": Path(x).expanduser().exists(), "files": len(files_of([x], None))}
118
+ for x in a.path + a.swebench]
119
+ return output(scan_files(files, searched, a.rules), a)
120
+ return output(scan.run(readers.default_roots(), a.since, a.rules), a)
121
+
122
+
123
+ if __name__ == "__main__":
124
+ sys.exit(main())