agent-hotwash 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. agent_hotwash-0.1.0/.github/workflows/ci.yml +29 -0
  2. agent_hotwash-0.1.0/.gitignore +218 -0
  3. agent_hotwash-0.1.0/.python-version +1 -0
  4. agent_hotwash-0.1.0/CHANGELOG.md +14 -0
  5. agent_hotwash-0.1.0/CLAUDE.md +37 -0
  6. agent_hotwash-0.1.0/FINDINGS.md +113 -0
  7. agent_hotwash-0.1.0/LICENSE +21 -0
  8. agent_hotwash-0.1.0/Makefile +116 -0
  9. agent_hotwash-0.1.0/PKG-INFO +94 -0
  10. agent_hotwash-0.1.0/README.md +82 -0
  11. agent_hotwash-0.1.0/config/defaults.toml +241 -0
  12. agent_hotwash-0.1.0/docs/research/.gitignore +2 -0
  13. agent_hotwash-0.1.0/docs/research/DESIGN.md +491 -0
  14. agent_hotwash-0.1.0/docs/research/claude_research.md +124 -0
  15. agent_hotwash-0.1.0/docs/research/format_findings.md +279 -0
  16. agent_hotwash-0.1.0/docs/using-agent-hotwash-for-coding-trace-analysis/SKILL.md +112 -0
  17. agent_hotwash-0.1.0/pyproject.toml +53 -0
  18. agent_hotwash-0.1.0/src/agent_hotwash/__init__.py +3 -0
  19. agent_hotwash-0.1.0/src/agent_hotwash/aggregate.py +194 -0
  20. agent_hotwash-0.1.0/src/agent_hotwash/analytics.py +498 -0
  21. agent_hotwash-0.1.0/src/agent_hotwash/cli.py +218 -0
  22. agent_hotwash-0.1.0/src/agent_hotwash/config.py +153 -0
  23. agent_hotwash-0.1.0/src/agent_hotwash/detectors/__init__.py +38 -0
  24. agent_hotwash-0.1.0/src/agent_hotwash/detectors/registry.py +312 -0
  25. agent_hotwash-0.1.0/src/agent_hotwash/detectors/smells.py +428 -0
  26. agent_hotwash-0.1.0/src/agent_hotwash/detectors/taxonomy.py +1089 -0
  27. agent_hotwash-0.1.0/src/agent_hotwash/events.py +185 -0
  28. agent_hotwash-0.1.0/src/agent_hotwash/primitives/__init__.py +6 -0
  29. agent_hotwash-0.1.0/src/agent_hotwash/primitives/argnorm.py +62 -0
  30. agent_hotwash-0.1.0/src/agent_hotwash/primitives/commands.py +155 -0
  31. agent_hotwash-0.1.0/src/agent_hotwash/primitives/errors.py +152 -0
  32. agent_hotwash-0.1.0/src/agent_hotwash/primitives/filestate.py +68 -0
  33. agent_hotwash-0.1.0/src/agent_hotwash/primitives/lexicons.py +71 -0
  34. agent_hotwash-0.1.0/src/agent_hotwash/primitives/outcome.py +151 -0
  35. agent_hotwash-0.1.0/src/agent_hotwash/primitives/window.py +66 -0
  36. agent_hotwash-0.1.0/src/agent_hotwash/report/__init__.py +22 -0
  37. agent_hotwash-0.1.0/src/agent_hotwash/report/csv_writer.py +95 -0
  38. agent_hotwash-0.1.0/src/agent_hotwash/report/html.py +216 -0
  39. agent_hotwash-0.1.0/src/agent_hotwash/report/json_writer.py +31 -0
  40. agent_hotwash-0.1.0/src/agent_hotwash/report/model.py +113 -0
  41. agent_hotwash-0.1.0/src/agent_hotwash/report/table.py +138 -0
  42. agent_hotwash-0.1.0/src/agent_hotwash/sources/__init__.py +6 -0
  43. agent_hotwash-0.1.0/src/agent_hotwash/sources/_common.py +377 -0
  44. agent_hotwash-0.1.0/src/agent_hotwash/sources/claude_native.py +263 -0
  45. agent_hotwash-0.1.0/src/agent_hotwash/sources/codebench.py +476 -0
  46. agent_hotwash-0.1.0/src/agent_hotwash/sources/codex_native.py +228 -0
  47. agent_hotwash-0.1.0/src/agent_hotwash/sources/detect.py +129 -0
  48. agent_hotwash-0.1.0/src/agent_hotwash/sources/pi_native.py +166 -0
  49. agent_hotwash-0.1.0/tests/conftest.py +278 -0
  50. agent_hotwash-0.1.0/tests/fixtures/claude_native/proj/sess-fixture/subagents/agent-sub-abc.jsonl +3 -0
  51. agent_hotwash-0.1.0/tests/fixtures/claude_native/proj/sess-fixture/subagents/agent-sub-abc.meta.json +1 -0
  52. agent_hotwash-0.1.0/tests/fixtures/claude_native/proj/sess-fixture.jsonl +11 -0
  53. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run/metrics.json +1 -0
  54. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run/run.json +1 -0
  55. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run/stdout.jsonl +12 -0
  56. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run/verification.json +1 -0
  57. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_multiblock/metrics.json +1 -0
  58. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_multiblock/run.json +1 -0
  59. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_multiblock/stdout.jsonl +6 -0
  60. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_multiblock/verification.json +1 -0
  61. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_no_usage/metrics.json +1 -0
  62. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_no_usage/run.json +1 -0
  63. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_no_usage/stdout.jsonl +5 -0
  64. agent_hotwash-0.1.0/tests/fixtures/codebench/claude_run_no_usage/verification.json +1 -0
  65. agent_hotwash-0.1.0/tests/fixtures/codebench/codex_run/metrics.json +1 -0
  66. agent_hotwash-0.1.0/tests/fixtures/codebench/codex_run/run.json +1 -0
  67. agent_hotwash-0.1.0/tests/fixtures/codebench/codex_run/stdout.jsonl +10 -0
  68. agent_hotwash-0.1.0/tests/fixtures/codebench/codex_run/verification.json +1 -0
  69. agent_hotwash-0.1.0/tests/fixtures/codebench/pi_run_zerousage/metrics.json +1 -0
  70. agent_hotwash-0.1.0/tests/fixtures/codebench/pi_run_zerousage/run.json +1 -0
  71. agent_hotwash-0.1.0/tests/fixtures/codebench/pi_run_zerousage/stdout.jsonl +13 -0
  72. agent_hotwash-0.1.0/tests/fixtures/codebench/pi_run_zerousage/verification.json +1 -0
  73. agent_hotwash-0.1.0/tests/fixtures/codex_native/rollout-fixture.jsonl +15 -0
  74. agent_hotwash-0.1.0/tests/fixtures/pi_native/session-fixture.jsonl +9 -0
  75. agent_hotwash-0.1.0/tests/learning/__init__.py +0 -0
  76. agent_hotwash-0.1.0/tests/learning/_helpers.py +61 -0
  77. agent_hotwash-0.1.0/tests/learning/test_learn_codebench_claude.py +116 -0
  78. agent_hotwash-0.1.0/tests/learning/test_learn_codebench_codex.py +101 -0
  79. agent_hotwash-0.1.0/tests/learning/test_learn_codebench_pi.py +138 -0
  80. agent_hotwash-0.1.0/tests/learning/test_learn_native_claude.py +146 -0
  81. agent_hotwash-0.1.0/tests/learning/test_learn_native_codex.py +150 -0
  82. agent_hotwash-0.1.0/tests/test_aggregate.py +134 -0
  83. agent_hotwash-0.1.0/tests/test_analytics.py +277 -0
  84. agent_hotwash-0.1.0/tests/test_argnorm.py +40 -0
  85. agent_hotwash-0.1.0/tests/test_cli.py +140 -0
  86. agent_hotwash-0.1.0/tests/test_commands.py +55 -0
  87. agent_hotwash-0.1.0/tests/test_common.py +112 -0
  88. agent_hotwash-0.1.0/tests/test_config.py +86 -0
  89. agent_hotwash-0.1.0/tests/test_detectors_smells.py +134 -0
  90. agent_hotwash-0.1.0/tests/test_detectors_taxonomy.py +332 -0
  91. agent_hotwash-0.1.0/tests/test_errors.py +147 -0
  92. agent_hotwash-0.1.0/tests/test_events.py +51 -0
  93. agent_hotwash-0.1.0/tests/test_filestate.py +47 -0
  94. agent_hotwash-0.1.0/tests/test_lexicons.py +52 -0
  95. agent_hotwash-0.1.0/tests/test_outcome.py +105 -0
  96. agent_hotwash-0.1.0/tests/test_registry.py +129 -0
  97. agent_hotwash-0.1.0/tests/test_report.py +136 -0
  98. agent_hotwash-0.1.0/tests/test_smoke.py +16 -0
  99. agent_hotwash-0.1.0/tests/test_sources_claude_native.py +86 -0
  100. agent_hotwash-0.1.0/tests/test_sources_codebench.py +194 -0
  101. agent_hotwash-0.1.0/tests/test_sources_codex_native.py +103 -0
  102. agent_hotwash-0.1.0/tests/test_sources_detect.py +160 -0
  103. agent_hotwash-0.1.0/tests/test_sources_pi_native.py +91 -0
  104. agent_hotwash-0.1.0/tests/test_window.py +46 -0
  105. agent_hotwash-0.1.0/uv.lock +319 -0
@@ -0,0 +1,29 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ concurrency:
9
+ group: ${{ github.workflow }}-${{ github.ref }}
10
+ cancel-in-progress: true
11
+
12
+ jobs:
13
+ check:
14
+ runs-on: ubuntu-latest
15
+ env:
16
+ NO_COLOR: "1"
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+
20
+ - name: Install uv
21
+ uses: astral-sh/setup-uv@v5
22
+ with:
23
+ enable-cache: true
24
+
25
+ - name: Install dependencies
26
+ run: make install
27
+
28
+ - name: Run checks
29
+ run: make check
@@ -0,0 +1,218 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ # Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ # poetry.lock
109
+ # poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ # pdm.lock
116
+ # pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ # pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # Redis
135
+ *.rdb
136
+ *.aof
137
+ *.pid
138
+
139
+ # RabbitMQ
140
+ mnesia/
141
+ rabbitmq/
142
+ rabbitmq-data/
143
+
144
+ # ActiveMQ
145
+ activemq-data/
146
+
147
+ # SageMath parsed files
148
+ *.sage.py
149
+
150
+ # Environments
151
+ .env
152
+ .envrc
153
+ .venv
154
+ env/
155
+ venv/
156
+ ENV/
157
+ env.bak/
158
+ venv.bak/
159
+
160
+ # Spyder project settings
161
+ .spyderproject
162
+ .spyproject
163
+
164
+ # Rope project settings
165
+ .ropeproject
166
+
167
+ # mkdocs documentation
168
+ /site
169
+
170
+ # mypy
171
+ .mypy_cache/
172
+ .dmypy.json
173
+ dmypy.json
174
+
175
+ # Pyre type checker
176
+ .pyre/
177
+
178
+ # pytype static type analyzer
179
+ .pytype/
180
+
181
+ # Cython debug symbols
182
+ cython_debug/
183
+
184
+ # PyCharm
185
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
186
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
187
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
188
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
189
+ # .idea/
190
+
191
+ # Abstra
192
+ # Abstra is an AI-powered process automation framework.
193
+ # Ignore directories containing user credentials, local state, and settings.
194
+ # Learn more at https://abstra.io/docs
195
+ .abstra/
196
+
197
+ # Visual Studio Code
198
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
199
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
200
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
201
+ # you could uncomment the following to ignore the entire vscode folder
202
+ # .vscode/
203
+ # Temporary file for partial code execution
204
+ tempCodeRunnerFile.py
205
+
206
+ # Ruff stuff:
207
+ .ruff_cache/
208
+
209
+ # PyPI configuration file
210
+ .pypirc
211
+
212
+ # Marimo
213
+ marimo/_static/
214
+ marimo/_lsp/
215
+ __marimo__/
216
+
217
+ # Streamlit
218
+ .streamlit/secrets.toml
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,14 @@
1
+ # Changelog
2
+
3
+ Semver. Each release gets a short, user-facing note: what changed for someone *using* the platform (operators, API consumers, deployers), not internal refactors. Keep entries minimal — one line where possible, grouped under `Added` / `Changed` / `Fixed` / `Removed` only when needed.
4
+
5
+ ## [Unreleased]
6
+
7
+ ### Added
8
+
9
+ ### Changed
10
+
11
+ ## [0.1.0] — 2026-07-05
12
+
13
+ ### Added
14
+ - Initial Release
@@ -0,0 +1,37 @@
1
+ # CLAUDE.md
2
+
3
+ `agent-hotwash` analyzes coding-agent traces (code-bench runs, native Claude
4
+ project dirs, native Codex rollouts) to surface improvement opportunities and
5
+ bad patterns. Python 3.12, managed with `uv`.
6
+
7
+ See `README.md` for usage/CLI and `docs/DESIGN.md` for architecture — don't
8
+ duplicate them here.
9
+
10
+ ## Git policy
11
+
12
+ **Never make any git changes (commit, push, tag, branch, rebase, etc.) unless
13
+ the user explicitly requests it.** Leave the working tree for the user to
14
+ review; when work is done, report what changed and stop.
15
+
16
+ ## Layout
17
+
18
+ - `src/agent_hotwash/cli.py` — CLI entrypoint (`agent-hotwash`).
19
+ - `src/agent_hotwash/sources/` — trace loaders + format autodetect (`detect.py`).
20
+ - `src/agent_hotwash/primitives/` — shared trace primitives (windows, file state, errors, commands).
21
+ - `src/agent_hotwash/analytics.py`, `aggregate.py`, `events.py` — analytics pipeline.
22
+ - `src/agent_hotwash/detectors/` — pattern detectors + registry/taxonomy.
23
+ - `src/agent_hotwash/report/` — output renderers (table/json/csv/html).
24
+ - `src/agent_hotwash/config.py`, `config/defaults.toml` — config + defaults.
25
+ - `tests/` — pytest suite. `docs/` — design notes and research.
26
+
27
+ ## Checks — use the Makefile
28
+
29
+ Always run checks via the Makefile targets. They are **quiet on success** (one
30
+ `<Name>: OK` line, full tool output only on failure) to keep transcripts short.
31
+
32
+ - `make check` — full gate: format-check, lint, typecheck, tests (parallel). Use before finishing work.
33
+ - `make lint` / `make format-check` / `make typecheck` / `make test` — individual legs.
34
+ - `make format` — apply ruff formatting (mutates files).
35
+ - `make install` — sync the venv from `uv.lock`. `make help` — list all targets.
36
+
37
+ Prefer these over calling `uv run ruff`/`pytest`/`ty` directly.
@@ -0,0 +1,113 @@
1
+ # Code-Bench Trace Findings
2
+
3
+ Analysis of `window-shop-monorepo-clean/tools/code-bench/runs` (293 runs, ~5.5 GB, 24 experiments)
4
+ using agent-hotwash + targeted subagent investigation. Every quantitative claim below survived a
5
+ second adversarial verification pass against the raw `run.json` / `verification.json` files;
6
+ claims that failed verification were corrected or dropped (noted inline).
7
+
8
+ Date: 2026-07-05
9
+
10
+ ## Headline: why codex (gpt-5.5) scored below opus-4-8
11
+
12
+ Verified resolved counts on the main 15-task × 3-repeat baselines:
13
+
14
+ | Experiment | Harness / model | Resolved |
15
+ |---|---|---|
16
+ | baseline-pi-opus48-xhigh | pi / claude-opus-4-8 | 35/45 |
17
+ | baseline-opus48-xhigh | claude / claude-opus-4-8 | 33/45 |
18
+ | baseline-codex-gpt55-xhigh | codex / gpt-5.5 | 29/45 |
19
+
20
+ Per-instance matrix (resolved out of 3 repeats; only divergent rows shown — the other 8 tasks are 3/3 everywhere):
21
+
22
+ | Instance | Codex | Opus (claude) | Opus (pi) |
23
+ |---|---|---|---|
24
+ | agents-3724 | 0 | 3 | 2 |
25
+ | wire-worker-3200 | 0 | 3 | 3 |
26
+ | pilot-web-3460 | 0 | 0 | 3 |
27
+ | checkout-2264 | **3** | 0 | 0 |
28
+ | checkout-3217 | 2 | 3 | 3 |
29
+ | agents-2885 | 0 | 0 | 0 |
30
+ | agents-3452 | 0 | 0 | 0 |
31
+
32
+ Root causes, ranked by evidence strength:
33
+
34
+ 1. **Over-refactoring breaks test contracts.** On `agents-3724`, codex refactored
35
+ `getOutfit(id, userId)` into object-style params (283-line patch); the fix logic was right but
36
+ two test mocks expecting the positional signature failed
37
+ (`baseline-codex-gpt55-xhigh/agents-3724/20260705T112502Z-codex-r3-878`). Opus made the same fix
38
+ keeping positional params (218 lines) and passed
39
+ (`baseline-opus48-xhigh/agents-3724/20260704T193740Z-claude-r3-504`). Same pattern on
40
+ `wire-worker-3200`: codex introduced a `SessionIdentity`/`PersistedSessionState` interface
41
+ hierarchy (264 lines) breaking 5 state-hydration tests; opus added properties directly
42
+ (148 lines) and passed. Codex ships ~30–80% more churn for the same fix.
43
+ 2. **No re-iteration after failure.** Failing codex runs average 256s vs ~252s for passing ones —
44
+ codex declares done and exits without looping back when tests fail. Opus averages ~615s
45
+ (2.4× longer) and spends most on its failures. (Earlier duration figures of 344s/972s were
46
+ overstated ~30%; corrected here.)
47
+ 3. **Speed cuts both ways.** `checkout-2264` is a codex-only win (3/3 vs opus 0/3) — a simple,
48
+ localized fix where codex's fast path wins and opus overworks (see "most expensive run" below).
49
+ 4. Zero timeouts in any of the three baselines — the gap is not time pressure.
50
+
51
+ ## The most surprising finding: same model, opposite outcome by harness
52
+
53
+ **pilot-web-3460** (session event buffering task): claude-opus-4-8 goes **0/3 under the Claude Code
54
+ harness but 3/3 under the pi harness**. Root cause is a single code-pattern choice, not the model's
55
+ task understanding:
56
+
57
+ - Failing claude-harness patches wrapped the store subscription in an `IS_CLIENT_SIDE` guard,
58
+ which interacts badly with vitest's `vi.doMock()` / module-reset lifecycle — the flush listener
59
+ never registers on the mocked store, failing 4/7 tests (FIFO order, buffer cap, reconnect).
60
+ - Repeat r2 then **overcorrected** by inventing an `sdkStatus === "connected"` precondition that
61
+ appears nowhere in the spec → 0/7.
62
+ - Passing pi-harness patches register the subscription unconditionally at module scope → 7/7.
63
+
64
+ One `if` statement cost three resolved runs. Codex also failed this task 0/3 (imperative
65
+ buffer-and-flush architecture mismatching the test's expectations).
66
+
67
+ ## Other verified anomalies
68
+
69
+ - **Most expensive run in the archive**: opus-4-8 spent **$12.05, 2028s, 15.9M tokens, 122 tool
70
+ calls** on `checkout-2264` — and failed on 1 of 94 tests
71
+ (`baseline-opus48-xhigh/checkout-2264/20260704T200316Z-claude-r3-026`).
72
+ - **Largest token run**: glm-5.2 burned **18.5M tokens, 187 tool calls, $5.80** — and still
73
+ resolved (`baseline-pi-glm52-xhigh-throughput/checkout-2264/20260705T151954Z-pi-r1-888`).
74
+ - **Silent regressions**: exactly 4 runs broke previously-passing tests (`pass_to_pass_regressed`),
75
+ e.g. an opus run on `checkout-3846` changed `recordVtoCompletion` from 1 arg to 2 in code it
76
+ wasn't asked to modify.
77
+ - **`timed_out` semantics look inconsistent**: raw-field count shows 100/293 runs flagged
78
+ `timed_out=true`, including 87/88 glm-5.2-xhigh runs — yet glm baselines still resolved ~27/44,
79
+ and a per-baseline check of the three headline experiments found zero timeouts. Worth checking
80
+ whether `timed_out` means hard-kill vs soft budget in the harness; the two analyses could not be
81
+ fully reconciled.
82
+ - **Unsolvable tasks**: `agents-2885` and `agents-3452` fail 0/3 across all three harnesses —
83
+ candidates for task-spec review.
84
+ - **Totals**: $530.79 spend across 293 runs, 210/293 (71.7%) resolved overall. (An earlier
85
+ $89.47 / 190-285 figure was wrong and is superseded.)
86
+ - Cost-per-resolved comparisons across models (e.g. haiku $0.026 vs opus $3.52) are
87
+ **apples-to-oranges** — cheap-model runs are smoke/toy tasks, not the 15-task baseline. Do not
88
+ quote them as model efficiency.
89
+
90
+ ## Library validation (agent-hotwash vs these traces)
91
+
92
+ The tool parsed all 136 baseline traces (claude, codex, pi) with exit 0, zero crashes, and
93
+ token/cost figures matching source `metrics.json` exactly on spot checks. Issues found and
94
+ **fixed on this branch** (independently re-validated, `make check` green):
95
+
96
+ 1. ~~Codex adapter metrics-blind~~ (file-op metrics all zero, every tool "other") — root cause was
97
+ a greedy `rglob("rollout-*.jsonl")` in `sources/detect.py` that descended into each code-bench
98
+ run's internal `traces/codex/sessions/…` rollout copy, so codex experiment dirs were parsed by
99
+ the *native* adapter instead of the codebench path. One fix cleared four symptoms.
100
+ 2. ~~45 run dirs → 46 traces~~ — the extra trace was that internal rollout copy, not a real second
101
+ session. Now exactly 45.
102
+ 3. ~~Cost labeled `estimated`~~ — now `provenance`, using the recorded `metrics.json` cost.
103
+ 4. ~~Codex parsing ~25× slower~~ — now 0.43s for 45 runs (was ~78s), faster than the claude path.
104
+ 5. Native codex tool names (`exec_command`, `apply_patch`, `read_file`, `update_plan`) now map to
105
+ execute/write/read/planning, with file paths recovered from `apply_patch` body headers.
106
+ 6. claude-native: `isMeta` user records (hook output, injected agent messages, local-command
107
+ caveats) were counted as genuine user turns — now dropped (14 sessions in the sampled project
108
+ dir had phantom turns).
109
+ 7. The reported `ty` typecheck failure in `tests/test_sources_claude_native.py` was not
110
+ reproducible (guarded comprehension narrows correctly); gate is green.
111
+
112
+ Both native traces validated post-fix: a codex rollout (860 events, execute 205 / write 36,
113
+ 1.24M input tokens) and a Claude Code session (171 events, 64 tool calls, zero warnings).
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 J S
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,116 @@
1
+ # agent-hotwash -- uv-based dev tasks. Run `make` or `make help` for the list.
2
+ # Convention: a `## description` after a target is picked up by the help target.
3
+
4
+ # Shared lint/format/typecheck target set.
5
+ SRC := src tests
6
+
7
+ # Tool invocations live here once so the verbose targets and the quiet `check`
8
+ # gate run byte-identical commands.
9
+ PYTEST := uv run pytest
10
+ TEST_CMD := $(PYTEST)
11
+ LINT_CMD := uv run ruff check $(SRC)
12
+ FMT_CHECK_CMD := uv run ruff format --check $(SRC)
13
+ TYPECHECK_CMD := uv run ty check --force-exclude $(SRC)
14
+
15
+ # ANSI colors for the check summary (export NO_COLOR=1 to disable).
16
+ ifndef NO_COLOR
17
+ GREEN := \033[32m
18
+ RED := \033[31m
19
+ BOLD := \033[1m
20
+ RESET := \033[0m
21
+ endif
22
+
23
+ # run_quiet: run a command with NO output on success (success = silence, the
24
+ # Unix way) and the full combined stdout+stderr on failure. Used by the
25
+ # standalone static-check targets (`make lint`, `make typecheck`, ...).
26
+ # usage: @$(call run_quiet,$(LINT_CMD))
27
+ define run_quiet
28
+ out=$$($(1) 2>&1); st=$$?; [ $$st -eq 0 ] || { printf '%s\n' "$$out"; exit $$st; }
29
+ endef
30
+
31
+ # run_check: like run_quiet but prints a one-line "<label>: OK" on success (and
32
+ # "<label>: FAIL" + the full output on failure). Used by the `make check` gate
33
+ # so each leg reports that it ran. Output is one atomic block, so it stays
34
+ # readable even when several run_check recipes run concurrently under `make -j`.
35
+ # usage: @$(call run_check,Lint,$(LINT_CMD))
36
+ define run_check
37
+ out=$$($(2) 2>&1); st=$$?; if [ $$st -eq 0 ]; then printf ' $(GREEN)%-14s OK$(RESET)\n' '$(1):'; else printf ' $(RED)%-14s FAIL$(RESET)\n' '$(1):'; printf '%s\n' "$$out"; exit $$st; fi
38
+ endef
39
+
40
+ .PHONY: help install sync format fmt lint format-check typecheck test check \
41
+ _ck-fmt _ck-lint _ck-type _ck-test release publish clean
42
+
43
+ help: ## Show this help (default target).
44
+ @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
45
+ | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}'
46
+
47
+ install: ## Sync the virtualenv from uv.lock (incl. dev group).
48
+ uv sync
49
+
50
+ sync: install ## Alias for install.
51
+
52
+ format: ## Format src + tests with ruff (mutates files).
53
+ uv run ruff format $(SRC)
54
+
55
+ fmt: format ## Alias for format.
56
+
57
+ # The standalone static-check targets are SILENT on success and dump the full
58
+ # combined stdout+stderr only when the tool exits non-zero. (`make check`
59
+ # reports each as "<name>: OK" via its own wrappers below.)
60
+ lint: ## Lint src + tests with ruff (silent unless it fails).
61
+ @$(call run_quiet,$(LINT_CMD))
62
+
63
+ format-check: ## Check formatting with ruff (silent unless it fails).
64
+ @$(call run_quiet,$(FMT_CHECK_CMD))
65
+
66
+ typecheck: ## Type-check src + tests with ty (silent unless it fails).
67
+ @$(call run_quiet,$(TYPECHECK_CMD))
68
+
69
+ test: ## Run the test suite (verbose).
70
+ $(TEST_CMD)
71
+
72
+ # check runs every leg via `make -j` so the static checks and tests run
73
+ # concurrently. Output is captured per target, so the summary lines never
74
+ # interleave. Quiet on success: each leg prints one "<name>: OK" line.
75
+ check: ## Full hygiene gate: format/lint/typecheck + tests (parallel). Quiet on success.
76
+ @printf '$(BOLD)Running make check...$(RESET)\n'
77
+ @$(MAKE) -j --no-print-directory _ck-fmt _ck-lint _ck-type _ck-test
78
+ @printf '$(GREEN)$(BOLD)All checks passed.$(RESET)\n'
79
+
80
+ _ck-fmt:
81
+ @$(call run_check,Format,$(FMT_CHECK_CMD))
82
+
83
+ _ck-lint:
84
+ @$(call run_check,Lint,$(LINT_CMD))
85
+
86
+ _ck-type:
87
+ @$(call run_check,Typecheck,$(TYPECHECK_CMD))
88
+
89
+ _ck-test:
90
+ @$(call run_check,Tests,$(TEST_CMD))
91
+
92
+ # release: bump the version, tag it, and build the wheel/sdist. Pass the bump
93
+ # level via BUMP (major|minor|patch; default patch), e.g. `make release BUMP=minor`.
94
+ # Runs the full check gate first so a broken tree never gets tagged.
95
+ BUMP ?= patch
96
+ release: check ## Bump version (BUMP=major|minor|patch), git-tag, and build dists.
97
+ @[ -z "$$(git status --porcelain)" ] || { echo "Release: FAIL (working tree not clean)"; exit 1; }
98
+ uv version --bump $(BUMP)
99
+ @version=$$(uv version --short); \
100
+ git commit -am "release: v$$version" && \
101
+ git tag "v$$version" && \
102
+ printf '$(GREEN)Tagged v%s$(RESET)\n' "$$version"
103
+ uv build
104
+
105
+ # publish: rebuild dists from scratch and upload with uv. Credentials via the
106
+ # usual uv env vars (UV_PUBLISH_TOKEN or UV_PUBLISH_USERNAME/PASSWORD).
107
+ publish: check ## Clean dist/, rebuild, and publish to PyPI with uv.
108
+ rm -rf dist
109
+ uv build
110
+ uv publish
111
+ rm -rf dist
112
+
113
+ clean: ## Remove caches and build artifacts (keeps .venv and uv.lock).
114
+ find . -type d -name __pycache__ -prune -exec rm -rf {} +
115
+ rm -rf .ruff_cache .pytest_cache dist build
116
+ find . -type f -name '*.pyc' -delete
@@ -0,0 +1,94 @@
1
+ Metadata-Version: 2.4
2
+ Name: agent-hotwash
3
+ Version: 0.1.0
4
+ Summary: agent-hotwash — analyze coding-agent traces to surface improvement opportunities and bad patterns
5
+ License-File: LICENSE
6
+ Requires-Python: <3.14,>=3.12
7
+ Requires-Dist: click>=8.4.2
8
+ Requires-Dist: pydantic>=2
9
+ Requires-Dist: rich
10
+ Requires-Dist: typer
11
+ Description-Content-Type: text/markdown
12
+
13
+ # agent-hotwash
14
+
15
+ Analyzes coding-agent traces (Claude, Codex, pi, code-bench) to surface bad patterns and improvement opportunities via analytics and detectors.
16
+ Point it at what an agent did to learn how it could have done better, with reports in table, JSON, CSV, or HTML for humans or CI.
17
+
18
+ ## Quickstart
19
+
20
+ Requires [uv](https://docs.astral.sh/uv/) and Python 3.12.
21
+
22
+ ```bash
23
+ uv sync # create the venv and install deps (incl. dev tools)
24
+ make check # run the full hygiene gate (format, lint, typecheck, tests)
25
+ uv run agent-hotwash --help
26
+ ```
27
+
28
+ ## Usage
29
+
30
+ Point `analyze` at any trace file or directory. It auto-detects the format
31
+ (code-bench run/experiment dirs, native Claude project dirs, native Codex
32
+ rollouts, native pi session dirs), runs analytics + detectors, and renders a
33
+ report.
34
+
35
+ ```bash
36
+ uv run agent-hotwash analyze <path>... # analyze one or more traces
37
+ uv run agent-hotwash analyze <path> --format table # human table (default on a TTY)
38
+ uv run agent-hotwash analyze <path> --format json # machine JSON (default when piped)
39
+ uv run agent-hotwash analyze <path> --format html --out report.html
40
+ uv run agent-hotwash analyze <path> --format csv --out rows.csv
41
+ ```
42
+
43
+ Common flags: `--config FILE` (user TOML merged over defaults), `--no-detectors`
44
+ (analytics only), `--fail-on high` (CI gate: non-zero exit if a finding at/above
45
+ the given severity is present).
46
+
47
+ Formats: `table` (rich, degrades to plain text off a TTY), `json`, `csv` (one
48
+ row per run + a column per finding id), `html` (single self-contained file, no
49
+ external assets). `--out` takes a file or a directory (writes `report.<ext>`);
50
+ omit it to stream to stdout.
51
+
52
+ Other commands:
53
+
54
+ ```bash
55
+ uv run agent-hotwash detectors --format json # list registered detectors
56
+ uv run agent-hotwash config-show # dump the effective merged config
57
+ uv run agent-hotwash version
58
+ ```
59
+
60
+ **Output contract (AI-friendly):** structured data goes to **stdout**, all
61
+ human/progress messages to **stderr**. Exit codes: `0` success, `1` error,
62
+ `2` no analyzable traces found (or `--fail-on` tripped).
63
+
64
+ ## Make targets
65
+
66
+ Run `make help` for the full list. The static-check targets are **quiet on
67
+ success** — they print one `<Name>: OK` line and only dump full tool output on
68
+ failure, to keep transcripts short for AI agents.
69
+
70
+ | Target | What it does |
71
+ | ------------ | ------------------------------------------------------------- |
72
+ | `install` / `sync` | Sync the venv from `uv.lock` (incl. dev group). |
73
+ | `format` / `fmt` | Format `src` + `tests` with ruff. |
74
+ | `lint` | Lint with ruff (silent unless it fails). |
75
+ | `format-check` | Check formatting with ruff (silent unless it fails). |
76
+ | `typecheck` | Type-check with [ty](https://github.com/astral-sh/ty). |
77
+ | `test` | Run the pytest suite (verbose). |
78
+ | `check` | Full gate: format-check + lint + typecheck + tests, parallel. |
79
+ | `release` | `make release BUMP=minor` — gate, bump, tag, build dists. |
80
+ | `clean` | Remove caches and build artifacts. |
81
+
82
+ ## Layout
83
+
84
+ ```
85
+ src/agent_hotwash/ package (CLI entry point in cli.py)
86
+ tests/ pytest suite
87
+ ```
88
+
89
+ ## Tooling
90
+
91
+ - **[uv](https://docs.astral.sh/uv/)** — env and dependency management.
92
+ - **[ruff](https://docs.astral.sh/ruff/)** — lint + format.
93
+ - **[ty](https://github.com/astral-sh/ty)** — type checking.
94
+ - **pytest** — tests.