traceverdict 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. traceverdict-0.2.0/LICENSE +21 -0
  2. traceverdict-0.2.0/PKG-INFO +123 -0
  3. traceverdict-0.2.0/README.md +103 -0
  4. traceverdict-0.2.0/pyproject.toml +46 -0
  5. traceverdict-0.2.0/setup.cfg +4 -0
  6. traceverdict-0.2.0/src/traceverdict/__init__.py +3 -0
  7. traceverdict-0.2.0/src/traceverdict/adapters/__init__.py +1 -0
  8. traceverdict-0.2.0/src/traceverdict/adapters/codex.py +235 -0
  9. traceverdict-0.2.0/src/traceverdict/adapters/mini_swe_agent.py +365 -0
  10. traceverdict-0.2.0/src/traceverdict/adapters/swe_agent.py +437 -0
  11. traceverdict-0.2.0/src/traceverdict/adapters/swe_agent_entrypoint.py +119 -0
  12. traceverdict-0.2.0/src/traceverdict/cli.py +309 -0
  13. traceverdict-0.2.0/src/traceverdict/compare/__init__.py +314 -0
  14. traceverdict-0.2.0/src/traceverdict/compare/constants.py +12 -0
  15. traceverdict-0.2.0/src/traceverdict/core/__init__.py +1 -0
  16. traceverdict-0.2.0/src/traceverdict/core/config_loader.py +72 -0
  17. traceverdict-0.2.0/src/traceverdict/core/runner.py +570 -0
  18. traceverdict-0.2.0/src/traceverdict/core/selftest.py +393 -0
  19. traceverdict-0.2.0/src/traceverdict/core/simple_yaml.py +252 -0
  20. traceverdict-0.2.0/src/traceverdict/core/suite.py +77 -0
  21. traceverdict-0.2.0/src/traceverdict/core/task_loader.py +69 -0
  22. traceverdict-0.2.0/src/traceverdict/cost_ledger.py +47 -0
  23. traceverdict-0.2.0/src/traceverdict/daily.py +397 -0
  24. traceverdict-0.2.0/src/traceverdict/ingest.py +272 -0
  25. traceverdict-0.2.0/src/traceverdict/injections/__init__.py +134 -0
  26. traceverdict-0.2.0/src/traceverdict/m3.py +484 -0
  27. traceverdict-0.2.0/src/traceverdict/m4c.py +134 -0
  28. traceverdict-0.2.0/src/traceverdict/m4d.py +381 -0
  29. traceverdict-0.2.0/src/traceverdict/m4s.py +154 -0
  30. traceverdict-0.2.0/src/traceverdict/report/__init__.py +128 -0
  31. traceverdict-0.2.0/src/traceverdict/report/taxonomy.py +76 -0
  32. traceverdict-0.2.0/src/traceverdict/resources/daily/configs/dev.yaml +15 -0
  33. traceverdict-0.2.0/src/traceverdict/resources/daily/configs/litellm_models.json +18 -0
  34. traceverdict-0.2.0/src/traceverdict/resources/daily/configs/litellm_models.meta.json +30 -0
  35. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/BASE_COMMIT.txt +1 -0
  36. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/repo.bundle +0 -0
  37. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/task.yaml +26 -0
  38. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/verify/README.md +11 -0
  39. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/BASE_COMMIT.txt +1 -0
  40. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/repo.bundle +0 -0
  41. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/task.yaml +24 -0
  42. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/verify/README.md +1 -0
  43. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/BASE_COMMIT.txt +1 -0
  44. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/repo.bundle +0 -0
  45. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/task.yaml +24 -0
  46. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/verify/README.md +1 -0
  47. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/BASE_COMMIT.txt +1 -0
  48. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/repo.bundle +0 -0
  49. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/task.yaml +24 -0
  50. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/verify/README.md +1 -0
  51. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/BASE_COMMIT.txt +1 -0
  52. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/repo.bundle +0 -0
  53. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/task.yaml +26 -0
  54. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/verify/README.md +1 -0
  55. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/BASE_COMMIT.txt +1 -0
  56. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/repo.bundle +0 -0
  57. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/task.yaml +29 -0
  58. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/verify/README.md +3 -0
  59. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/BASE_COMMIT.txt +1 -0
  60. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/repo.bundle +0 -0
  61. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/task.yaml +22 -0
  62. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/verify/README.md +1 -0
  63. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/BASE_COMMIT.txt +1 -0
  64. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/repo.bundle +0 -0
  65. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/task.yaml +24 -0
  66. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/verify/README.md +1 -0
  67. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/Dockerfile +7 -0
  68. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/SWEAgent.Dockerfile +10 -0
  69. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/SWEAgentV2.Dockerfile +11 -0
  70. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/SWEAgentV3.Dockerfile +10 -0
  71. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/metadata.json +10 -0
  72. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/requirements.txt +3 -0
  73. traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/task_set.txt +8 -0
  74. traceverdict-0.2.0/src/traceverdict/resources.py +49 -0
  75. traceverdict-0.2.0/src/traceverdict/snapshot/__init__.py +1 -0
  76. traceverdict-0.2.0/src/traceverdict/snapshot/codex_image.py +143 -0
  77. traceverdict-0.2.0/src/traceverdict/snapshot/image.py +110 -0
  78. traceverdict-0.2.0/src/traceverdict/snapshot/patch.py +36 -0
  79. traceverdict-0.2.0/src/traceverdict/snapshot/suite_image.py +62 -0
  80. traceverdict-0.2.0/src/traceverdict/snapshot/workspace.py +113 -0
  81. traceverdict-0.2.0/src/traceverdict/swebench_adapter.py +508 -0
  82. traceverdict-0.2.0/src/traceverdict/swebench_budget.py +114 -0
  83. traceverdict-0.2.0/src/traceverdict/tracer/__init__.py +1 -0
  84. traceverdict-0.2.0/src/traceverdict/tracer/codex_jsonl.py +238 -0
  85. traceverdict-0.2.0/src/traceverdict/tracer/db.py +238 -0
  86. traceverdict-0.2.0/src/traceverdict/tracer/schema.sql +93 -0
  87. traceverdict-0.2.0/src/traceverdict/tracer/trajectory.py +422 -0
  88. traceverdict-0.2.0/src/traceverdict/verifier/__init__.py +213 -0
  89. traceverdict-0.2.0/src/traceverdict.egg-info/PKG-INFO +123 -0
  90. traceverdict-0.2.0/src/traceverdict.egg-info/SOURCES.txt +124 -0
  91. traceverdict-0.2.0/src/traceverdict.egg-info/dependency_links.txt +1 -0
  92. traceverdict-0.2.0/src/traceverdict.egg-info/entry_points.txt +3 -0
  93. traceverdict-0.2.0/src/traceverdict.egg-info/requires.txt +7 -0
  94. traceverdict-0.2.0/src/traceverdict.egg-info/top_level.txt +1 -0
  95. traceverdict-0.2.0/tests/test_adapter_guards.py +210 -0
  96. traceverdict-0.2.0/tests/test_cli.py +108 -0
  97. traceverdict-0.2.0/tests/test_codex_adapter.py +204 -0
  98. traceverdict-0.2.0/tests/test_compare_report.py +272 -0
  99. traceverdict-0.2.0/tests/test_daily.py +170 -0
  100. traceverdict-0.2.0/tests/test_daily_resources.py +26 -0
  101. traceverdict-0.2.0/tests/test_docker_require.py +16 -0
  102. traceverdict-0.2.0/tests/test_finalize_m3.py +193 -0
  103. traceverdict-0.2.0/tests/test_ingest.py +85 -0
  104. traceverdict-0.2.0/tests/test_injections.py +277 -0
  105. traceverdict-0.2.0/tests/test_m3.py +385 -0
  106. traceverdict-0.2.0/tests/test_m4c.py +78 -0
  107. traceverdict-0.2.0/tests/test_m4c_finalizer.py +35 -0
  108. traceverdict-0.2.0/tests/test_m4c_orchestrator.py +124 -0
  109. traceverdict-0.2.0/tests/test_m4d.py +120 -0
  110. traceverdict-0.2.0/tests/test_m4s.py +118 -0
  111. traceverdict-0.2.0/tests/test_paid_run_recovery.py +105 -0
  112. traceverdict-0.2.0/tests/test_public_safety.py +32 -0
  113. traceverdict-0.2.0/tests/test_runner_external_image.py +50 -0
  114. traceverdict-0.2.0/tests/test_runner_wall_time.py +27 -0
  115. traceverdict-0.2.0/tests/test_s1_task_load.py +205 -0
  116. traceverdict-0.2.0/tests/test_sample_swebv.py +93 -0
  117. traceverdict-0.2.0/tests/test_schema.py +149 -0
  118. traceverdict-0.2.0/tests/test_selftest.py +64 -0
  119. traceverdict-0.2.0/tests/test_snapshot.py +227 -0
  120. traceverdict-0.2.0/tests/test_suite.py +59 -0
  121. traceverdict-0.2.0/tests/test_swe_agent_adapter.py +304 -0
  122. traceverdict-0.2.0/tests/test_swebench_adapter.py +410 -0
  123. traceverdict-0.2.0/tests/test_t5_pilot.py +231 -0
  124. traceverdict-0.2.0/tests/test_taxonomy.py +65 -0
  125. traceverdict-0.2.0/tests/test_trajectory.py +412 -0
  126. traceverdict-0.2.0/tests/test_verifier.py +177 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 weaqaq
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,123 @@
1
+ Metadata-Version: 2.4
2
+ Name: traceverdict
3
+ Version: 0.2.0
4
+ Summary: Auditable state-regression evaluation for coding agents; published by the project owner
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/weaqaq/traceverdict
7
+ Project-URL: Repository, https://github.com/weaqaq/traceverdict.git
8
+ Project-URL: Issues, https://github.com/weaqaq/traceverdict/issues
9
+ Requires-Python: >=3.12
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: typer
13
+ Requires-Dist: pydantic
14
+ Requires-Dist: docker
15
+ Requires-Dist: litellm
16
+ Requires-Dist: rich
17
+ Requires-Dist: pytest
18
+ Requires-Dist: GitPython
19
+ Dynamic: license-file
20
+
21
+ # TraceVerdict
22
+
23
+ TraceVerdict is an auditable regression-evaluation harness for coding agents. It freezes repository and container state, captures native agent trajectories, and compares the resulting file changes and official test verdicts with paired statistics.
24
+
25
+ TraceVerdict is not a general-purpose call recorder or agent-rewind system: its unit of evidence is a reproducible coding task with a frozen workspace, environment fingerprint, authoritative patch, and test outcome. It sits below general observability products such as LangSmith and Langfuse as a vertical state-regression layer for coding agents; traces can be exported to those systems. It runs on top of the official SWE-bench harness, whose verdict remains ground truth—TraceVerdict records, reconciles, and compares that verdict rather than replacing the harness.
26
+
27
+ > This repository is a clean public mirror. Full per-commit audit history and raw run data remain in a private audit repository and can be shown in an interview. Public documents preserve the decision and finding narrative while removing machine paths, credentials, personal information, and infrastructure details.
28
+
29
+ ## What it provides
30
+
31
+ - adapters for mini-swe-agent 2.4.5 and pinned SWE-agent 1.1.0;
32
+ - immutable config identities and strict provider usage/cost reconciliation;
33
+ - disposable Docker workspaces with patch, artifact, and image-digest evidence;
34
+ - SWE-bench 4.1.0 official-harness verdict integration;
35
+ - paired 10,000-resample bootstrap, exact McNemar, regression alarms, and failure taxonomy;
36
+ - deterministic comparison/report generation with explicit shared-sample disclosure;
37
+ - faithful fault injection that isolates the agent environment from the verifier.
38
+
39
+ ## Evidence, with limits
40
+
41
+ M3 froze 16 SWE-bench Verified tasks and added 85 paid evaluations. The cross-provider experiment used 16 tasks at `k=2`; its pass delta was `-0.09375` with bootstrap 95% CI `[-0.21875, 0.03125]`. This is engineering evidence for the regression pipeline, not a broad model leaderboard claim. A read-only-workspace regression produced a hard alarm (delta `-0.3125`, CI `[-0.5, -0.125]`) with 32/32 agent-read-only/verifier-read-write isolation records. Total audited API spend was `$1.8627937123` against a `$28` tripwire.
42
+
43
+ See the [M3 evidence summary](docs/evidence/m3/m3_summary.md) and its [machine-readable companion](docs/evidence/m3/m3_summary.json).
44
+
45
+ The later [M4-C compatibility arm](docs/evidence/m4c/m4c_codex_arm_20260718.md) ran a pinned Codex CLI locally while keeping subscription credentials off the verifier host. It completed 16/16 official judgments with 6/16 resolved; versus the DeepSeek Flash `k=2` baseline, the authorized asymmetric comparison yielded delta pass `+0.0625`, 95% CI `[-0.15625, 0.3125]`, exact McNemar `p=0.25`, and no regression alarm. This is cross-scaffolding compatibility evidence, not a model leaderboard claim.
46
+
47
+ ## Install
48
+
49
+ Python 3.12 and Docker are required for real task execution.
50
+
51
+ ```bash
52
+ python -m venv .venv
53
+ python -m pip install -e .
54
+ python -m pip install "mini-swe-agent==2.4.5"
55
+ traceverdict --help
56
+ ```
57
+
58
+ TraceVerdict v0.2 exposes exactly ten top-level commands through either `traceverdict` or the equivalent short entry point `tv`:
59
+
60
+ ```text
61
+ traceverdict run TASK --config CONFIG
62
+ traceverdict suite tasks/self --config configs/dev.yaml --dry-run
63
+ traceverdict compare --baseline BASE --candidate CANDIDATE --task-set tasks/self/task_set.txt
64
+ traceverdict report COMPARISON_ID
65
+ traceverdict inject I3Q --base configs/dev.yaml --output injected.yaml
66
+ traceverdict replay
67
+ traceverdict selftest --config configs/dev.yaml
68
+ tv quick --set model=openai/deepseek-v4-flash
69
+ tv baseline set --config configs/dev.yaml
70
+ tv ingest
71
+ ```
72
+
73
+ `replay` remains an intentionally visible zero-model CI boundary and currently exits with code 2. Paid/provider runs require credentials supplied only through the process environment; no credential file belongs in this repository.
74
+
75
+ ## Daily Mode
76
+
77
+ Daily Mode keeps its local, ignored state under `.traceverdict/daily/`. First establish an explicit cached baseline (this is the only step that may execute its missing baseline tasks):
78
+
79
+ ```bash
80
+ tv baseline set --config configs/dev.yaml
81
+ tv quick --set model=openai/deepseek-v4-flash
82
+ tv quick --set model=openai/deepseek-v4-flash --set model_params.thinking.type=enabled --full
83
+ ```
84
+
85
+ The default smoke set is frozen to S1/S4/S6; `--full` is frozen to S1-S8 and reuses already completed smoke runs. SWE-bench is deliberately unavailable through `quick`: the 16-task public benchmark is for release evidence, not daily iteration. Derived configs are content-addressed, exact-price-registry checked, and immutable. `baseline update` promotes an already complete candidate without running a model, and refuses correctness or forbidden-path regressions unless `--accept-regression` is explicit.
86
+
87
+ The one-screen result reports pass delta, token-median delta/ratio, wall-P95 delta/ratio, strict actual cost, cache reuse, and failed tasks. PASS/WARN exit 0, a behavioral FAIL exits 1, and a missing baseline or invalid identity exits 2. The five-minute/$0.005 and full-suite timing figures are experience estimates only; each invocation reports measured time and cost.
88
+
89
+ `tv ingest [PATH ...]` is passive and starts no model, Docker container, or verifier. It incrementally summarizes stable `codex exec --json` logs and the explicitly versioned July 2026 desktop-rollout compatibility format. Only dates, model, token classes, turns, tool counts, and failure classes are persisted; prompts, answers, commands, output, paths, and credentials are not. Desktop transcript format is not a stable public interface, so required-field drift fails closed instead of being guessed.
90
+
91
+ This package is published by the project owner. Trusted Publishing through GitHub Actions OIDC is planned as a v0.3 supply-chain improvement.
92
+
93
+ ## Reproduce the frozen report
94
+
95
+ ```bash
96
+ python scripts/finalize_m3.py \
97
+ --db reports/m3/traceverdict.db \
98
+ --completion-root reports/m3 \
99
+ --output reports/m3/final
100
+ ```
101
+
102
+ The command fails closed on an incomplete 16-task/`k=2` matrix, missing metrics or verdicts, environment-fingerprint drift, missing I3Q isolation evidence, or an incomplete SWE-agent trace. Raw databases and trajectories are intentionally absent from the public mirror.
103
+
104
+ ## Findings index
105
+
106
+ - [F-1](docs/findings/F1.md): deleting tool instructions did not degrade the tested agent at `n=8, k=1`.
107
+ - [F-2](docs/findings/F2.md): truncating observations to 500 characters invited adaptive recovery.
108
+ - [F-3](docs/findings/F3.md): a task-template-loss alarm was unstable when it depended only on cost at `k=1`.
109
+ - [F-4](docs/findings/F4.md): CRLF conversion caused a cross-platform forbidden-path false positive.
110
+ - [F-5](docs/findings/F5.md): strict reconciliation caught a provider reasoning-token pricing partition bug.
111
+ - [F-6](docs/findings/F6.md): a 96-run, four-arm audit separated shared failures into sampling difficulty, runtime constraints, and submitted task-level convergence.
112
+
113
+ ## Build/reuse boundary
114
+
115
+ Self-built components are the tracer, agent adapters, verifier wiring, snapshot discipline, comparison statistics, taxonomy, injection isolation, and evidence/replay boundaries. Reused components are mini-swe-agent, SWE-agent, the SWE-bench official harness and images, LiteLLM, Docker, and provider APIs. See the [public PRD](docs/PRD.md), [decision log](docs/decisions.md), and [publication provenance](docs/publication_provenance.md).
116
+
117
+ ## Security and provenance
118
+
119
+ CI runs a fail-closed public-safety scan over text, binary bundles, nested Git authors/content, and repository history. The initial public history is one genuine release commit; it does not imitate the private development history. See [name audit](docs/name_audit.md).
120
+
121
+ ## License
122
+
123
+ [MIT](LICENSE)
@@ -0,0 +1,103 @@
1
+ # TraceVerdict
2
+
3
+ TraceVerdict is an auditable regression-evaluation harness for coding agents. It freezes repository and container state, captures native agent trajectories, and compares the resulting file changes and official test verdicts with paired statistics.
4
+
5
+ TraceVerdict is not a general-purpose call recorder or agent-rewind system: its unit of evidence is a reproducible coding task with a frozen workspace, environment fingerprint, authoritative patch, and test outcome. It sits below general observability products such as LangSmith and Langfuse as a vertical state-regression layer for coding agents; traces can be exported to those systems. It runs on top of the official SWE-bench harness, whose verdict remains ground truth—TraceVerdict records, reconciles, and compares that verdict rather than replacing the harness.
6
+
7
+ > This repository is a clean public mirror. Full per-commit audit history and raw run data remain in a private audit repository and can be shown in an interview. Public documents preserve the decision and finding narrative while removing machine paths, credentials, personal information, and infrastructure details.
8
+
9
+ ## What it provides
10
+
11
+ - adapters for mini-swe-agent 2.4.5 and pinned SWE-agent 1.1.0;
12
+ - immutable config identities and strict provider usage/cost reconciliation;
13
+ - disposable Docker workspaces with patch, artifact, and image-digest evidence;
14
+ - SWE-bench 4.1.0 official-harness verdict integration;
15
+ - paired 10,000-resample bootstrap, exact McNemar, regression alarms, and failure taxonomy;
16
+ - deterministic comparison/report generation with explicit shared-sample disclosure;
17
+ - faithful fault injection that isolates the agent environment from the verifier.
18
+
19
+ ## Evidence, with limits
20
+
21
+ M3 froze 16 SWE-bench Verified tasks and added 85 paid evaluations. The cross-provider experiment used 16 tasks at `k=2`; its pass delta was `-0.09375` with bootstrap 95% CI `[-0.21875, 0.03125]`. This is engineering evidence for the regression pipeline, not a broad model leaderboard claim. A read-only-workspace regression produced a hard alarm (delta `-0.3125`, CI `[-0.5, -0.125]`) with 32/32 agent-read-only/verifier-read-write isolation records. Total audited API spend was `$1.8627937123` against a `$28` tripwire.
22
+
23
+ See the [M3 evidence summary](docs/evidence/m3/m3_summary.md) and its [machine-readable companion](docs/evidence/m3/m3_summary.json).
24
+
25
+ The later [M4-C compatibility arm](docs/evidence/m4c/m4c_codex_arm_20260718.md) ran a pinned Codex CLI locally while keeping subscription credentials off the verifier host. It completed 16/16 official judgments with 6/16 resolved; versus the DeepSeek Flash `k=2` baseline, the authorized asymmetric comparison yielded delta pass `+0.0625`, 95% CI `[-0.15625, 0.3125]`, exact McNemar `p=0.25`, and no regression alarm. This is cross-scaffolding compatibility evidence, not a model leaderboard claim.
26
+
27
+ ## Install
28
+
29
+ Python 3.12 and Docker are required for real task execution.
30
+
31
+ ```bash
32
+ python -m venv .venv
33
+ python -m pip install -e .
34
+ python -m pip install "mini-swe-agent==2.4.5"
35
+ traceverdict --help
36
+ ```
37
+
38
+ TraceVerdict v0.2 exposes exactly ten top-level commands through either `traceverdict` or the equivalent short entry point `tv`:
39
+
40
+ ```text
41
+ traceverdict run TASK --config CONFIG
42
+ traceverdict suite tasks/self --config configs/dev.yaml --dry-run
43
+ traceverdict compare --baseline BASE --candidate CANDIDATE --task-set tasks/self/task_set.txt
44
+ traceverdict report COMPARISON_ID
45
+ traceverdict inject I3Q --base configs/dev.yaml --output injected.yaml
46
+ traceverdict replay
47
+ traceverdict selftest --config configs/dev.yaml
48
+ tv quick --set model=openai/deepseek-v4-flash
49
+ tv baseline set --config configs/dev.yaml
50
+ tv ingest
51
+ ```
52
+
53
+ `replay` remains an intentionally visible zero-model CI boundary and currently exits with code 2. Paid/provider runs require credentials supplied only through the process environment; no credential file belongs in this repository.
54
+
55
+ ## Daily Mode
56
+
57
+ Daily Mode keeps its local, ignored state under `.traceverdict/daily/`. First establish an explicit cached baseline (this is the only step that may execute its missing baseline tasks):
58
+
59
+ ```bash
60
+ tv baseline set --config configs/dev.yaml
61
+ tv quick --set model=openai/deepseek-v4-flash
62
+ tv quick --set model=openai/deepseek-v4-flash --set model_params.thinking.type=enabled --full
63
+ ```
64
+
65
+ The default smoke set is frozen to S1/S4/S6; `--full` is frozen to S1-S8 and reuses already completed smoke runs. SWE-bench is deliberately unavailable through `quick`: the 16-task public benchmark is for release evidence, not daily iteration. Derived configs are content-addressed, exact-price-registry checked, and immutable. `baseline update` promotes an already complete candidate without running a model, and refuses correctness or forbidden-path regressions unless `--accept-regression` is explicit.
66
+
67
+ The one-screen result reports pass delta, token-median delta/ratio, wall-P95 delta/ratio, strict actual cost, cache reuse, and failed tasks. PASS/WARN exit 0, a behavioral FAIL exits 1, and a missing baseline or invalid identity exits 2. The five-minute/$0.005 and full-suite timing figures are experience estimates only; each invocation reports measured time and cost.
68
+
69
+ `tv ingest [PATH ...]` is passive and starts no model, Docker container, or verifier. It incrementally summarizes stable `codex exec --json` logs and the explicitly versioned July 2026 desktop-rollout compatibility format. Only dates, model, token classes, turns, tool counts, and failure classes are persisted; prompts, answers, commands, output, paths, and credentials are not. Desktop transcript format is not a stable public interface, so required-field drift fails closed instead of being guessed.
70
+
71
+ This package is published by the project owner. Trusted Publishing through GitHub Actions OIDC is planned as a v0.3 supply-chain improvement.
72
+
73
+ ## Reproduce the frozen report
74
+
75
+ ```bash
76
+ python scripts/finalize_m3.py \
77
+ --db reports/m3/traceverdict.db \
78
+ --completion-root reports/m3 \
79
+ --output reports/m3/final
80
+ ```
81
+
82
+ The command fails closed on an incomplete 16-task/`k=2` matrix, missing metrics or verdicts, environment-fingerprint drift, missing I3Q isolation evidence, or an incomplete SWE-agent trace. Raw databases and trajectories are intentionally absent from the public mirror.
83
+
84
+ ## Findings index
85
+
86
+ - [F-1](docs/findings/F1.md): deleting tool instructions did not degrade the tested agent at `n=8, k=1`.
87
+ - [F-2](docs/findings/F2.md): truncating observations to 500 characters invited adaptive recovery.
88
+ - [F-3](docs/findings/F3.md): a task-template-loss alarm was unstable when it depended only on cost at `k=1`.
89
+ - [F-4](docs/findings/F4.md): CRLF conversion caused a cross-platform forbidden-path false positive.
90
+ - [F-5](docs/findings/F5.md): strict reconciliation caught a provider reasoning-token pricing partition bug.
91
+ - [F-6](docs/findings/F6.md): a 96-run, four-arm audit separated shared failures into sampling difficulty, runtime constraints, and submitted task-level convergence.
92
+
93
+ ## Build/reuse boundary
94
+
95
+ Self-built components are the tracer, agent adapters, verifier wiring, snapshot discipline, comparison statistics, taxonomy, injection isolation, and evidence/replay boundaries. Reused components are mini-swe-agent, SWE-agent, the SWE-bench official harness and images, LiteLLM, Docker, and provider APIs. See the [public PRD](docs/PRD.md), [decision log](docs/decisions.md), and [publication provenance](docs/publication_provenance.md).
96
+
97
+ ## Security and provenance
98
+
99
+ CI runs a fail-closed public-safety scan over text, binary bundles, nested Git authors/content, and repository history. The initial public history is one genuine release commit; it does not imitate the private development history. See [name audit](docs/name_audit.md).
100
+
101
+ ## License
102
+
103
+ [MIT](LICENSE)
@@ -0,0 +1,46 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "traceverdict"
7
+ version = "0.2.0"
8
+ description = "Auditable state-regression evaluation for coding agents; published by the project owner"
9
+ readme = "README.md"
10
+ requires-python = ">=3.12"
11
+ license = "MIT"
12
+ dependencies = [
13
+ "typer",
14
+ "pydantic",
15
+ "docker",
16
+ "litellm",
17
+ "rich",
18
+ "pytest",
19
+ "GitPython",
20
+ ]
21
+
22
+ [project.scripts]
23
+ traceverdict = "traceverdict.cli:app"
24
+ tv = "traceverdict.cli:app"
25
+
26
+ [project.urls]
27
+ Homepage = "https://github.com/weaqaq/traceverdict"
28
+ Repository = "https://github.com/weaqaq/traceverdict.git"
29
+ Issues = "https://github.com/weaqaq/traceverdict/issues"
30
+
31
+ [tool.setuptools.packages.find]
32
+ where = ["src"]
33
+
34
+ [tool.setuptools.package-data]
35
+ traceverdict = [
36
+ "tracer/schema.sql",
37
+ "resources/daily/configs/*",
38
+ "resources/daily/tasks/self/*",
39
+ "resources/daily/tasks/self/_image/*",
40
+ "resources/daily/tasks/self/S*/*",
41
+ "resources/daily/tasks/self/S*/verify/*",
42
+ ]
43
+
44
+ [tool.pytest.ini_options]
45
+ testpaths = ["tests"]
46
+ pythonpath = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """TraceVerdict: stateful regression evaluation system for coding agents."""
2
+
3
+ __version__ = "0.2.0"
@@ -0,0 +1 @@
1
+ """adapters: integration layer for agents under test (mini_swe_agent first)."""
@@ -0,0 +1,235 @@
1
+ """Local-only Codex exec adapter for the M4-C compatibility arm (D24)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import subprocess
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from traceverdict.adapters.mini_swe_agent import AdapterHarnessError, AdapterResult
13
+ from traceverdict.swebench_budget import ADAPTER_WALL_TIMEOUT_GRACE_S
14
+
15
+ PINNED_CODEX_VERSION = "0.144.4"
16
+ CODEX_TRAJECTORY_FORMAT = "codex-exec-jsonl-1"
17
+ CODEX_BINARY_IN_IMAGE = "/opt/traceverdict/codex"
18
+
19
+
20
+ def _config_args(model_params: dict[str, Any]) -> list[str]:
21
+ """Return the complete behavior-affecting Codex config overrides."""
22
+ effort = model_params.get("model_reasoning_effort")
23
+ if effort not in {"high", "medium"}:
24
+ raise AdapterHarnessError(
25
+ "model_reasoning_effort must be an explicitly approved M4-C value "
26
+ f"(high or medium), got {effort!r}"
27
+ )
28
+ expected = {
29
+ "auth_mode": "chatgpt_subscription",
30
+ "billing_mode": "subscription_unallocatable",
31
+ "approval_policy": "never",
32
+ "sandbox_mode": "workspace-write",
33
+ "web_search": "disabled",
34
+ "ephemeral": True,
35
+ "ignore_user_config": True,
36
+ "ignore_rules": True,
37
+ "concurrency": 1,
38
+ "network_mode": "bridge",
39
+ "container_cap_add": "SYS_ADMIN",
40
+ "container_seccomp": "unconfined",
41
+ "inner_sandbox": "workspace-write",
42
+ "auth_isolation": "per_tool_mount_namespace_hide_codex_home_v2",
43
+ }
44
+ mismatches = {
45
+ key: {"expected": value, "actual": model_params.get(key)}
46
+ for key, value in expected.items()
47
+ if model_params.get(key) != value
48
+ }
49
+ if mismatches:
50
+ raise AdapterHarnessError(
51
+ "immutable Codex M4-C identity mismatch: "
52
+ + json.dumps(mismatches, sort_keys=True)
53
+ )
54
+ nested_expected = {
55
+ "sandbox_workspace_write": {"network_access": True},
56
+ "history": {"persistence": "none"},
57
+ "features": {
58
+ "shell_tool": True,
59
+ "shell_snapshot": False,
60
+ "unified_exec": False,
61
+ },
62
+ "hide_agent_reasoning": False,
63
+ }
64
+ nested_mismatches = {
65
+ key: {"expected": value, "actual": model_params.get(key)}
66
+ for key, value in nested_expected.items()
67
+ if model_params.get(key) != value
68
+ }
69
+ if nested_mismatches:
70
+ raise AdapterHarnessError(
71
+ "immutable Codex nested identity mismatch: "
72
+ + json.dumps(nested_mismatches, sort_keys=True)
73
+ )
74
+ if model_params.get("service_tier") != "omitted":
75
+ raise AdapterHarnessError("service_tier must be explicitly omitted")
76
+ if model_params.get("temperature") != "omitted" or model_params.get("top_p") != "omitted":
77
+ raise AdapterHarnessError("temperature and top_p must be explicitly omitted")
78
+ return [
79
+ "-c", f"model_reasoning_effort=\"{effort}\"",
80
+ "-c", "approval_policy=\"never\"",
81
+ "-c", "web_search=\"disabled\"",
82
+ "-c", "history.persistence=\"none\"",
83
+ "-c", "hide_agent_reasoning=false",
84
+ "-c", "features.shell_tool=true",
85
+ "-c", "features.shell_snapshot=false",
86
+ "-c", "features.unified_exec=false",
87
+ "-c", "sandbox_workspace_write.network_access=true",
88
+ ]
89
+
90
+
91
+ def build_codex_command(
92
+ *, model_name: str, model_params: dict[str, Any], container_cwd: str
93
+ ) -> list[str]:
94
+ return [
95
+ CODEX_BINARY_IN_IMAGE,
96
+ "exec",
97
+ "--json",
98
+ "--ephemeral",
99
+ "--ignore-user-config",
100
+ "--ignore-rules",
101
+ "--strict-config",
102
+ "--model",
103
+ model_name,
104
+ "--sandbox",
105
+ "workspace-write",
106
+ "-C",
107
+ container_cwd,
108
+ *_config_args(model_params),
109
+ "-",
110
+ ]
111
+
112
+
113
+ def parse_codex_jsonl(data: str) -> list[dict[str, Any]]:
114
+ records: list[dict[str, Any]] = []
115
+ for line_number, raw in enumerate(data.splitlines(), start=1):
116
+ if not raw.strip():
117
+ continue
118
+ try:
119
+ item = json.loads(raw)
120
+ except json.JSONDecodeError as exc:
121
+ raise AdapterHarnessError(
122
+ f"Codex JSONL line {line_number} is invalid: {exc}"
123
+ ) from exc
124
+ if not isinstance(item, dict) or not isinstance(item.get("type"), str):
125
+ raise AdapterHarnessError(
126
+ f"Codex JSONL line {line_number} lacks an event type"
127
+ )
128
+ records.append(item)
129
+ if not records:
130
+ raise AdapterHarnessError("Codex produced no JSONL records")
131
+ return records
132
+
133
+
134
+ def run_codex(
135
+ *,
136
+ instruction: str,
137
+ image: str,
138
+ docker_executable: str,
139
+ host_work_path: Path,
140
+ container_cwd: str,
141
+ model_name: str,
142
+ model_params: dict[str, Any],
143
+ litellm_model_registry: Path | None,
144
+ agent_version: str,
145
+ cost_limit: float,
146
+ step_limit: int,
147
+ wall_time_s: int,
148
+ work_dir: Path,
149
+ ) -> AdapterResult:
150
+ """Run Codex inside the local agent-layer image; never use a remote host."""
151
+ del litellm_model_registry, cost_limit, step_limit
152
+ if agent_version != PINNED_CODEX_VERSION:
153
+ raise AdapterHarnessError(
154
+ f"Codex version mismatch: expected {PINNED_CODEX_VERSION}, got {agent_version}"
155
+ )
156
+ auth_spec = os.environ.get("TRACEVERDICT_CODEX_AUTH_FILE")
157
+ if not auth_spec:
158
+ raise AdapterHarnessError("TRACEVERDICT_CODEX_AUTH_FILE is required locally")
159
+ auth_file = Path(auth_spec).resolve()
160
+ if not auth_file.is_file():
161
+ raise AdapterHarnessError("local Codex auth file does not exist")
162
+ work_dir.mkdir(parents=True, exist_ok=True)
163
+ jsonl_path = work_dir / "codex.events.jsonl"
164
+ stderr_path = work_dir / "codex.stderr.log"
165
+ command = build_codex_command(
166
+ model_name=model_name,
167
+ model_params=model_params,
168
+ container_cwd=container_cwd,
169
+ )
170
+ host_work = str(host_work_path.resolve()).replace("\\", "/")
171
+ host_auth = str(auth_file).replace("\\", "/")
172
+ docker_command = [
173
+ docker_executable,
174
+ "run",
175
+ "--rm",
176
+ "-i",
177
+ "--network",
178
+ "bridge",
179
+ "--cap-add",
180
+ "SYS_ADMIN",
181
+ "--security-opt",
182
+ "seccomp=unconfined",
183
+ "-e",
184
+ "CODEX_HOME=/run/traceverdict-codex",
185
+ "-v",
186
+ f"{host_work}:{container_cwd}",
187
+ "-v",
188
+ f"{host_auth}:/run/traceverdict-codex/auth.json",
189
+ "-w",
190
+ container_cwd,
191
+ image,
192
+ *command,
193
+ ]
194
+ timeout = max(1, int(wall_time_s)) + ADAPTER_WALL_TIMEOUT_GRACE_S
195
+ try:
196
+ proc = subprocess.run(
197
+ docker_command,
198
+ input=instruction,
199
+ capture_output=True,
200
+ text=True,
201
+ encoding="utf-8",
202
+ errors="strict",
203
+ timeout=timeout,
204
+ check=False,
205
+ )
206
+ except subprocess.TimeoutExpired as exc:
207
+ raise AdapterHarnessError(
208
+ f"Codex exceeded adapter timeout {timeout}s", exit_reason="TimeExceeded"
209
+ ) from exc
210
+ jsonl_path.write_text(proc.stdout or "", encoding="utf-8")
211
+ stderr_path.write_text(proc.stderr or "", encoding="utf-8")
212
+ records = parse_codex_jsonl(proc.stdout or "")
213
+ trajectory = {
214
+ "trajectory_format": CODEX_TRAJECTORY_FORMAT,
215
+ "records": records,
216
+ "jsonl_sha256": hashlib.sha256((proc.stdout or "").encode("utf-8")).hexdigest(),
217
+ "instruction_sha256": hashlib.sha256(instruction.encode("utf-8")).hexdigest(),
218
+ "info": {
219
+ "submission": "",
220
+ "model_stats": {"instance_cost": None},
221
+ "returncode": proc.returncode,
222
+ },
223
+ }
224
+ trajectory_path = work_dir / "codex.trajectory.json"
225
+ trajectory_path.write_text(
226
+ json.dumps(trajectory, ensure_ascii=False, indent=2), encoding="utf-8"
227
+ )
228
+ return AdapterResult(
229
+ traj=trajectory,
230
+ traj_path=trajectory_path,
231
+ returncode=proc.returncode,
232
+ stdout=proc.stdout or "",
233
+ stderr=proc.stderr or "",
234
+ source_artifacts={"codex_jsonl": jsonl_path},
235
+ )