traceverdict 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- traceverdict-0.2.0/LICENSE +21 -0
- traceverdict-0.2.0/PKG-INFO +123 -0
- traceverdict-0.2.0/README.md +103 -0
- traceverdict-0.2.0/pyproject.toml +46 -0
- traceverdict-0.2.0/setup.cfg +4 -0
- traceverdict-0.2.0/src/traceverdict/__init__.py +3 -0
- traceverdict-0.2.0/src/traceverdict/adapters/__init__.py +1 -0
- traceverdict-0.2.0/src/traceverdict/adapters/codex.py +235 -0
- traceverdict-0.2.0/src/traceverdict/adapters/mini_swe_agent.py +365 -0
- traceverdict-0.2.0/src/traceverdict/adapters/swe_agent.py +437 -0
- traceverdict-0.2.0/src/traceverdict/adapters/swe_agent_entrypoint.py +119 -0
- traceverdict-0.2.0/src/traceverdict/cli.py +309 -0
- traceverdict-0.2.0/src/traceverdict/compare/__init__.py +314 -0
- traceverdict-0.2.0/src/traceverdict/compare/constants.py +12 -0
- traceverdict-0.2.0/src/traceverdict/core/__init__.py +1 -0
- traceverdict-0.2.0/src/traceverdict/core/config_loader.py +72 -0
- traceverdict-0.2.0/src/traceverdict/core/runner.py +570 -0
- traceverdict-0.2.0/src/traceverdict/core/selftest.py +393 -0
- traceverdict-0.2.0/src/traceverdict/core/simple_yaml.py +252 -0
- traceverdict-0.2.0/src/traceverdict/core/suite.py +77 -0
- traceverdict-0.2.0/src/traceverdict/core/task_loader.py +69 -0
- traceverdict-0.2.0/src/traceverdict/cost_ledger.py +47 -0
- traceverdict-0.2.0/src/traceverdict/daily.py +397 -0
- traceverdict-0.2.0/src/traceverdict/ingest.py +272 -0
- traceverdict-0.2.0/src/traceverdict/injections/__init__.py +134 -0
- traceverdict-0.2.0/src/traceverdict/m3.py +484 -0
- traceverdict-0.2.0/src/traceverdict/m4c.py +134 -0
- traceverdict-0.2.0/src/traceverdict/m4d.py +381 -0
- traceverdict-0.2.0/src/traceverdict/m4s.py +154 -0
- traceverdict-0.2.0/src/traceverdict/report/__init__.py +128 -0
- traceverdict-0.2.0/src/traceverdict/report/taxonomy.py +76 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/configs/dev.yaml +15 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/configs/litellm_models.json +18 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/configs/litellm_models.meta.json +30 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/task.yaml +26 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S1/verify/README.md +11 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/task.yaml +24 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S2/verify/README.md +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/task.yaml +24 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S3/verify/README.md +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/task.yaml +24 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S4/verify/README.md +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/task.yaml +26 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S5/verify/README.md +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/task.yaml +29 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S6/verify/README.md +3 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/task.yaml +22 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S7/verify/README.md +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/BASE_COMMIT.txt +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/repo.bundle +0 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/task.yaml +24 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/S8/verify/README.md +1 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/Dockerfile +7 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/SWEAgent.Dockerfile +10 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/SWEAgentV2.Dockerfile +11 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/SWEAgentV3.Dockerfile +10 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/metadata.json +10 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/_image/requirements.txt +3 -0
- traceverdict-0.2.0/src/traceverdict/resources/daily/tasks/self/task_set.txt +8 -0
- traceverdict-0.2.0/src/traceverdict/resources.py +49 -0
- traceverdict-0.2.0/src/traceverdict/snapshot/__init__.py +1 -0
- traceverdict-0.2.0/src/traceverdict/snapshot/codex_image.py +143 -0
- traceverdict-0.2.0/src/traceverdict/snapshot/image.py +110 -0
- traceverdict-0.2.0/src/traceverdict/snapshot/patch.py +36 -0
- traceverdict-0.2.0/src/traceverdict/snapshot/suite_image.py +62 -0
- traceverdict-0.2.0/src/traceverdict/snapshot/workspace.py +113 -0
- traceverdict-0.2.0/src/traceverdict/swebench_adapter.py +508 -0
- traceverdict-0.2.0/src/traceverdict/swebench_budget.py +114 -0
- traceverdict-0.2.0/src/traceverdict/tracer/__init__.py +1 -0
- traceverdict-0.2.0/src/traceverdict/tracer/codex_jsonl.py +238 -0
- traceverdict-0.2.0/src/traceverdict/tracer/db.py +238 -0
- traceverdict-0.2.0/src/traceverdict/tracer/schema.sql +93 -0
- traceverdict-0.2.0/src/traceverdict/tracer/trajectory.py +422 -0
- traceverdict-0.2.0/src/traceverdict/verifier/__init__.py +213 -0
- traceverdict-0.2.0/src/traceverdict.egg-info/PKG-INFO +123 -0
- traceverdict-0.2.0/src/traceverdict.egg-info/SOURCES.txt +124 -0
- traceverdict-0.2.0/src/traceverdict.egg-info/dependency_links.txt +1 -0
- traceverdict-0.2.0/src/traceverdict.egg-info/entry_points.txt +3 -0
- traceverdict-0.2.0/src/traceverdict.egg-info/requires.txt +7 -0
- traceverdict-0.2.0/src/traceverdict.egg-info/top_level.txt +1 -0
- traceverdict-0.2.0/tests/test_adapter_guards.py +210 -0
- traceverdict-0.2.0/tests/test_cli.py +108 -0
- traceverdict-0.2.0/tests/test_codex_adapter.py +204 -0
- traceverdict-0.2.0/tests/test_compare_report.py +272 -0
- traceverdict-0.2.0/tests/test_daily.py +170 -0
- traceverdict-0.2.0/tests/test_daily_resources.py +26 -0
- traceverdict-0.2.0/tests/test_docker_require.py +16 -0
- traceverdict-0.2.0/tests/test_finalize_m3.py +193 -0
- traceverdict-0.2.0/tests/test_ingest.py +85 -0
- traceverdict-0.2.0/tests/test_injections.py +277 -0
- traceverdict-0.2.0/tests/test_m3.py +385 -0
- traceverdict-0.2.0/tests/test_m4c.py +78 -0
- traceverdict-0.2.0/tests/test_m4c_finalizer.py +35 -0
- traceverdict-0.2.0/tests/test_m4c_orchestrator.py +124 -0
- traceverdict-0.2.0/tests/test_m4d.py +120 -0
- traceverdict-0.2.0/tests/test_m4s.py +118 -0
- traceverdict-0.2.0/tests/test_paid_run_recovery.py +105 -0
- traceverdict-0.2.0/tests/test_public_safety.py +32 -0
- traceverdict-0.2.0/tests/test_runner_external_image.py +50 -0
- traceverdict-0.2.0/tests/test_runner_wall_time.py +27 -0
- traceverdict-0.2.0/tests/test_s1_task_load.py +205 -0
- traceverdict-0.2.0/tests/test_sample_swebv.py +93 -0
- traceverdict-0.2.0/tests/test_schema.py +149 -0
- traceverdict-0.2.0/tests/test_selftest.py +64 -0
- traceverdict-0.2.0/tests/test_snapshot.py +227 -0
- traceverdict-0.2.0/tests/test_suite.py +59 -0
- traceverdict-0.2.0/tests/test_swe_agent_adapter.py +304 -0
- traceverdict-0.2.0/tests/test_swebench_adapter.py +410 -0
- traceverdict-0.2.0/tests/test_t5_pilot.py +231 -0
- traceverdict-0.2.0/tests/test_taxonomy.py +65 -0
- traceverdict-0.2.0/tests/test_trajectory.py +412 -0
- traceverdict-0.2.0/tests/test_verifier.py +177 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 weaqaq
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: traceverdict
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Auditable state-regression evaluation for coding agents; published by the project owner
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/weaqaq/traceverdict
|
|
7
|
+
Project-URL: Repository, https://github.com/weaqaq/traceverdict.git
|
|
8
|
+
Project-URL: Issues, https://github.com/weaqaq/traceverdict/issues
|
|
9
|
+
Requires-Python: >=3.12
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: typer
|
|
13
|
+
Requires-Dist: pydantic
|
|
14
|
+
Requires-Dist: docker
|
|
15
|
+
Requires-Dist: litellm
|
|
16
|
+
Requires-Dist: rich
|
|
17
|
+
Requires-Dist: pytest
|
|
18
|
+
Requires-Dist: GitPython
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# TraceVerdict
|
|
22
|
+
|
|
23
|
+
TraceVerdict is an auditable regression-evaluation harness for coding agents. It freezes repository and container state, captures native agent trajectories, and compares the resulting file changes and official test verdicts with paired statistics.
|
|
24
|
+
|
|
25
|
+
TraceVerdict is not a general-purpose call recorder or agent-rewind system: its unit of evidence is a reproducible coding task with a frozen workspace, environment fingerprint, authoritative patch, and test outcome. It sits below general observability products such as LangSmith and Langfuse as a vertical state-regression layer for coding agents; traces can be exported to those systems. It runs on top of the official SWE-bench harness, whose verdict remains ground truth—TraceVerdict records, reconciles, and compares that verdict rather than replacing the harness.
|
|
26
|
+
|
|
27
|
+
> This repository is a clean public mirror. Full per-commit audit history and raw run data remain in a private audit repository and can be shown in an interview. Public documents preserve the decision and finding narrative while removing machine paths, credentials, personal information, and infrastructure details.
|
|
28
|
+
|
|
29
|
+
## What it provides
|
|
30
|
+
|
|
31
|
+
- adapters for mini-swe-agent 2.4.5 and pinned SWE-agent 1.1.0;
|
|
32
|
+
- immutable config identities and strict provider usage/cost reconciliation;
|
|
33
|
+
- disposable Docker workspaces with patch, artifact, and image-digest evidence;
|
|
34
|
+
- SWE-bench 4.1.0 official-harness verdict integration;
|
|
35
|
+
- paired 10,000-resample bootstrap, exact McNemar, regression alarms, and failure taxonomy;
|
|
36
|
+
- deterministic comparison/report generation with explicit shared-sample disclosure;
|
|
37
|
+
- faithful fault injection that isolates the agent environment from the verifier.
|
|
38
|
+
|
|
39
|
+
## Evidence, with limits
|
|
40
|
+
|
|
41
|
+
M3 froze 16 SWE-bench Verified tasks and added 85 paid evaluations. The cross-provider experiment used 16 tasks at `k=2`; its pass delta was `-0.09375` with bootstrap 95% CI `[-0.21875, 0.03125]`. This is engineering evidence for the regression pipeline, not a broad model leaderboard claim. A read-only-workspace regression produced a hard alarm (delta `-0.3125`, CI `[-0.5, -0.125]`) with 32/32 agent-read-only/verifier-read-write isolation records. Total audited API spend was `$1.8627937123` against a `$28` tripwire.
|
|
42
|
+
|
|
43
|
+
See the [M3 evidence summary](docs/evidence/m3/m3_summary.md) and its [machine-readable companion](docs/evidence/m3/m3_summary.json).
|
|
44
|
+
|
|
45
|
+
The later [M4-C compatibility arm](docs/evidence/m4c/m4c_codex_arm_20260718.md) ran a pinned Codex CLI locally while keeping subscription credentials off the verifier host. It completed 16/16 official judgments with 6/16 resolved; versus the DeepSeek Flash `k=2` baseline, the authorized asymmetric comparison yielded delta pass `+0.0625`, 95% CI `[-0.15625, 0.3125]`, exact McNemar `p=0.25`, and no regression alarm. This is cross-scaffolding compatibility evidence, not a model leaderboard claim.
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
Python 3.12 and Docker are required for real task execution.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python -m venv .venv
|
|
53
|
+
python -m pip install -e .
|
|
54
|
+
python -m pip install "mini-swe-agent==2.4.5"
|
|
55
|
+
traceverdict --help
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
TraceVerdict v0.2 exposes exactly ten top-level commands through either `traceverdict` or the equivalent short entry point `tv`:
|
|
59
|
+
|
|
60
|
+
```text
|
|
61
|
+
traceverdict run TASK --config CONFIG
|
|
62
|
+
traceverdict suite tasks/self --config configs/dev.yaml --dry-run
|
|
63
|
+
traceverdict compare --baseline BASE --candidate CANDIDATE --task-set tasks/self/task_set.txt
|
|
64
|
+
traceverdict report COMPARISON_ID
|
|
65
|
+
traceverdict inject I3Q --base configs/dev.yaml --output injected.yaml
|
|
66
|
+
traceverdict replay
|
|
67
|
+
traceverdict selftest --config configs/dev.yaml
|
|
68
|
+
tv quick --set model=openai/deepseek-v4-flash
|
|
69
|
+
tv baseline set --config configs/dev.yaml
|
|
70
|
+
tv ingest
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`replay` remains an intentionally visible zero-model CI boundary and currently exits with code 2. Paid/provider runs require credentials supplied only through the process environment; no credential file belongs in this repository.
|
|
74
|
+
|
|
75
|
+
## Daily Mode
|
|
76
|
+
|
|
77
|
+
Daily Mode keeps its local, ignored state under `.traceverdict/daily/`. First establish an explicit cached baseline (this is the only step that may execute its missing baseline tasks):
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
tv baseline set --config configs/dev.yaml
|
|
81
|
+
tv quick --set model=openai/deepseek-v4-flash
|
|
82
|
+
tv quick --set model=openai/deepseek-v4-flash --set model_params.thinking.type=enabled --full
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The default smoke set is frozen to S1/S4/S6; `--full` is frozen to S1-S8 and reuses already completed smoke runs. SWE-bench is deliberately unavailable through `quick`: the 16-task public benchmark is for release evidence, not daily iteration. Derived configs are content-addressed, exact-price-registry checked, and immutable. `baseline update` promotes an already complete candidate without running a model, and refuses correctness or forbidden-path regressions unless `--accept-regression` is explicit.
|
|
86
|
+
|
|
87
|
+
The one-screen result reports pass delta, token-median delta/ratio, wall-P95 delta/ratio, strict actual cost, cache reuse, and failed tasks. PASS/WARN exit 0, a behavioral FAIL exits 1, and a missing baseline or invalid identity exits 2. The five-minute/$0.005 and full-suite timing figures are experience estimates only; each invocation reports measured time and cost.
|
|
88
|
+
|
|
89
|
+
`tv ingest [PATH ...]` is passive and starts no model, Docker container, or verifier. It incrementally summarizes stable `codex exec --json` logs and the explicitly versioned July 2026 desktop-rollout compatibility format. Only dates, model, token classes, turns, tool counts, and failure classes are persisted; prompts, answers, commands, output, paths, and credentials are not. Desktop transcript format is not a stable public interface, so required-field drift fails closed instead of being guessed.
|
|
90
|
+
|
|
91
|
+
This package is published by the project owner. Trusted Publishing through GitHub Actions OIDC is planned as a v0.3 supply-chain improvement.
|
|
92
|
+
|
|
93
|
+
## Reproduce the frozen report
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
python scripts/finalize_m3.py \
|
|
97
|
+
--db reports/m3/traceverdict.db \
|
|
98
|
+
--completion-root reports/m3 \
|
|
99
|
+
--output reports/m3/final
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The command fails closed on an incomplete 16-task/`k=2` matrix, missing metrics or verdicts, environment-fingerprint drift, missing I3Q isolation evidence, or an incomplete SWE-agent trace. Raw databases and trajectories are intentionally absent from the public mirror.
|
|
103
|
+
|
|
104
|
+
## Findings index
|
|
105
|
+
|
|
106
|
+
- [F-1](docs/findings/F1.md): deleting tool instructions did not degrade the tested agent at `n=8, k=1`.
|
|
107
|
+
- [F-2](docs/findings/F2.md): truncating observations to 500 characters invited adaptive recovery.
|
|
108
|
+
- [F-3](docs/findings/F3.md): a task-template-loss alarm was unstable when it depended only on cost at `k=1`.
|
|
109
|
+
- [F-4](docs/findings/F4.md): CRLF conversion caused a cross-platform forbidden-path false positive.
|
|
110
|
+
- [F-5](docs/findings/F5.md): strict reconciliation caught a provider reasoning-token pricing partition bug.
|
|
111
|
+
- [F-6](docs/findings/F6.md): a 96-run, four-arm audit separated shared failures into sampling difficulty, runtime constraints, and submitted task-level convergence.
|
|
112
|
+
|
|
113
|
+
## Build/reuse boundary
|
|
114
|
+
|
|
115
|
+
Self-built components are the tracer, agent adapters, verifier wiring, snapshot discipline, comparison statistics, taxonomy, injection isolation, and evidence/replay boundaries. Reused components are mini-swe-agent, SWE-agent, the SWE-bench official harness and images, LiteLLM, Docker, and provider APIs. See the [public PRD](docs/PRD.md), [decision log](docs/decisions.md), and [publication provenance](docs/publication_provenance.md).
|
|
116
|
+
|
|
117
|
+
## Security and provenance
|
|
118
|
+
|
|
119
|
+
CI runs a fail-closed public-safety scan over text, binary bundles, nested Git authors/content, and repository history. The initial public history is one genuine release commit; it does not imitate the private development history. See [name audit](docs/name_audit.md).
|
|
120
|
+
|
|
121
|
+
## License
|
|
122
|
+
|
|
123
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# TraceVerdict
|
|
2
|
+
|
|
3
|
+
TraceVerdict is an auditable regression-evaluation harness for coding agents. It freezes repository and container state, captures native agent trajectories, and compares the resulting file changes and official test verdicts with paired statistics.
|
|
4
|
+
|
|
5
|
+
TraceVerdict is not a general-purpose call recorder or agent-rewind system: its unit of evidence is a reproducible coding task with a frozen workspace, environment fingerprint, authoritative patch, and test outcome. It sits below general observability products such as LangSmith and Langfuse as a vertical state-regression layer for coding agents; traces can be exported to those systems. It runs on top of the official SWE-bench harness, whose verdict remains ground truth—TraceVerdict records, reconciles, and compares that verdict rather than replacing the harness.
|
|
6
|
+
|
|
7
|
+
> This repository is a clean public mirror. Full per-commit audit history and raw run data remain in a private audit repository and can be shown in an interview. Public documents preserve the decision and finding narrative while removing machine paths, credentials, personal information, and infrastructure details.
|
|
8
|
+
|
|
9
|
+
## What it provides
|
|
10
|
+
|
|
11
|
+
- adapters for mini-swe-agent 2.4.5 and pinned SWE-agent 1.1.0;
|
|
12
|
+
- immutable config identities and strict provider usage/cost reconciliation;
|
|
13
|
+
- disposable Docker workspaces with patch, artifact, and image-digest evidence;
|
|
14
|
+
- SWE-bench 4.1.0 official-harness verdict integration;
|
|
15
|
+
- paired 10,000-resample bootstrap, exact McNemar, regression alarms, and failure taxonomy;
|
|
16
|
+
- deterministic comparison/report generation with explicit shared-sample disclosure;
|
|
17
|
+
- faithful fault injection that isolates the agent environment from the verifier.
|
|
18
|
+
|
|
19
|
+
## Evidence, with limits
|
|
20
|
+
|
|
21
|
+
M3 froze 16 SWE-bench Verified tasks and added 85 paid evaluations. The cross-provider experiment used 16 tasks at `k=2`; its pass delta was `-0.09375` with bootstrap 95% CI `[-0.21875, 0.03125]`. This is engineering evidence for the regression pipeline, not a broad model leaderboard claim. A read-only-workspace regression produced a hard alarm (delta `-0.3125`, CI `[-0.5, -0.125]`) with 32/32 agent-read-only/verifier-read-write isolation records. Total audited API spend was `$1.8627937123` against a `$28` tripwire.
|
|
22
|
+
|
|
23
|
+
See the [M3 evidence summary](docs/evidence/m3/m3_summary.md) and its [machine-readable companion](docs/evidence/m3/m3_summary.json).
|
|
24
|
+
|
|
25
|
+
The later [M4-C compatibility arm](docs/evidence/m4c/m4c_codex_arm_20260718.md) ran a pinned Codex CLI locally while keeping subscription credentials off the verifier host. It completed 16/16 official judgments with 6/16 resolved; versus the DeepSeek Flash `k=2` baseline, the authorized asymmetric comparison yielded delta pass `+0.0625`, 95% CI `[-0.15625, 0.3125]`, exact McNemar `p=0.25`, and no regression alarm. This is cross-scaffolding compatibility evidence, not a model leaderboard claim.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
Python 3.12 and Docker are required for real task execution.
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
python -m venv .venv
|
|
33
|
+
python -m pip install -e .
|
|
34
|
+
python -m pip install "mini-swe-agent==2.4.5"
|
|
35
|
+
traceverdict --help
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
TraceVerdict v0.2 exposes exactly ten top-level commands through either `traceverdict` or the equivalent short entry point `tv`:
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
traceverdict run TASK --config CONFIG
|
|
42
|
+
traceverdict suite tasks/self --config configs/dev.yaml --dry-run
|
|
43
|
+
traceverdict compare --baseline BASE --candidate CANDIDATE --task-set tasks/self/task_set.txt
|
|
44
|
+
traceverdict report COMPARISON_ID
|
|
45
|
+
traceverdict inject I3Q --base configs/dev.yaml --output injected.yaml
|
|
46
|
+
traceverdict replay
|
|
47
|
+
traceverdict selftest --config configs/dev.yaml
|
|
48
|
+
tv quick --set model=openai/deepseek-v4-flash
|
|
49
|
+
tv baseline set --config configs/dev.yaml
|
|
50
|
+
tv ingest
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
`replay` remains an intentionally visible zero-model CI boundary and currently exits with code 2. Paid/provider runs require credentials supplied only through the process environment; no credential file belongs in this repository.
|
|
54
|
+
|
|
55
|
+
## Daily Mode
|
|
56
|
+
|
|
57
|
+
Daily Mode keeps its local, ignored state under `.traceverdict/daily/`. First establish an explicit cached baseline (this is the only step that may execute its missing baseline tasks):
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
tv baseline set --config configs/dev.yaml
|
|
61
|
+
tv quick --set model=openai/deepseek-v4-flash
|
|
62
|
+
tv quick --set model=openai/deepseek-v4-flash --set model_params.thinking.type=enabled --full
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The default smoke set is frozen to S1/S4/S6; `--full` is frozen to S1-S8 and reuses already completed smoke runs. SWE-bench is deliberately unavailable through `quick`: the 16-task public benchmark is for release evidence, not daily iteration. Derived configs are content-addressed, exact-price-registry checked, and immutable. `baseline update` promotes an already complete candidate without running a model, and refuses correctness or forbidden-path regressions unless `--accept-regression` is explicit.
|
|
66
|
+
|
|
67
|
+
The one-screen result reports pass delta, token-median delta/ratio, wall-P95 delta/ratio, strict actual cost, cache reuse, and failed tasks. PASS/WARN exit 0, a behavioral FAIL exits 1, and a missing baseline or invalid identity exits 2. The five-minute/$0.005 and full-suite timing figures are experience estimates only; each invocation reports measured time and cost.
|
|
68
|
+
|
|
69
|
+
`tv ingest [PATH ...]` is passive and starts no model, Docker container, or verifier. It incrementally summarizes stable `codex exec --json` logs and the explicitly versioned July 2026 desktop-rollout compatibility format. Only dates, model, token classes, turns, tool counts, and failure classes are persisted; prompts, answers, commands, output, paths, and credentials are not. Desktop transcript format is not a stable public interface, so required-field drift fails closed instead of being guessed.
|
|
70
|
+
|
|
71
|
+
This package is published by the project owner. Trusted Publishing through GitHub Actions OIDC is planned as a v0.3 supply-chain improvement.
|
|
72
|
+
|
|
73
|
+
## Reproduce the frozen report
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
python scripts/finalize_m3.py \
|
|
77
|
+
--db reports/m3/traceverdict.db \
|
|
78
|
+
--completion-root reports/m3 \
|
|
79
|
+
--output reports/m3/final
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The command fails closed on an incomplete 16-task/`k=2` matrix, missing metrics or verdicts, environment-fingerprint drift, missing I3Q isolation evidence, or an incomplete SWE-agent trace. Raw databases and trajectories are intentionally absent from the public mirror.
|
|
83
|
+
|
|
84
|
+
## Findings index
|
|
85
|
+
|
|
86
|
+
- [F-1](docs/findings/F1.md): deleting tool instructions did not degrade the tested agent at `n=8, k=1`.
|
|
87
|
+
- [F-2](docs/findings/F2.md): truncating observations to 500 characters invited adaptive recovery.
|
|
88
|
+
- [F-3](docs/findings/F3.md): a task-template-loss alarm was unstable when it depended only on cost at `k=1`.
|
|
89
|
+
- [F-4](docs/findings/F4.md): CRLF conversion caused a cross-platform forbidden-path false positive.
|
|
90
|
+
- [F-5](docs/findings/F5.md): strict reconciliation caught a provider reasoning-token pricing partition bug.
|
|
91
|
+
- [F-6](docs/findings/F6.md): a 96-run, four-arm audit separated shared failures into sampling difficulty, runtime constraints, and submitted task-level convergence.
|
|
92
|
+
|
|
93
|
+
## Build/reuse boundary
|
|
94
|
+
|
|
95
|
+
Self-built components are the tracer, agent adapters, verifier wiring, snapshot discipline, comparison statistics, taxonomy, injection isolation, and evidence/replay boundaries. Reused components are mini-swe-agent, SWE-agent, the SWE-bench official harness and images, LiteLLM, Docker, and provider APIs. See the [public PRD](docs/PRD.md), [decision log](docs/decisions.md), and [publication provenance](docs/publication_provenance.md).
|
|
96
|
+
|
|
97
|
+
## Security and provenance
|
|
98
|
+
|
|
99
|
+
CI runs a fail-closed public-safety scan over text, binary bundles, nested Git authors/content, and repository history. The initial public history is one genuine release commit; it does not imitate the private development history. See [name audit](docs/name_audit.md).
|
|
100
|
+
|
|
101
|
+
## License
|
|
102
|
+
|
|
103
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "traceverdict"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Auditable state-regression evaluation for coding agents; published by the project owner"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"typer",
|
|
14
|
+
"pydantic",
|
|
15
|
+
"docker",
|
|
16
|
+
"litellm",
|
|
17
|
+
"rich",
|
|
18
|
+
"pytest",
|
|
19
|
+
"GitPython",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
traceverdict = "traceverdict.cli:app"
|
|
24
|
+
tv = "traceverdict.cli:app"
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/weaqaq/traceverdict"
|
|
28
|
+
Repository = "https://github.com/weaqaq/traceverdict.git"
|
|
29
|
+
Issues = "https://github.com/weaqaq/traceverdict/issues"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
|
|
34
|
+
[tool.setuptools.package-data]
|
|
35
|
+
traceverdict = [
|
|
36
|
+
"tracer/schema.sql",
|
|
37
|
+
"resources/daily/configs/*",
|
|
38
|
+
"resources/daily/tasks/self/*",
|
|
39
|
+
"resources/daily/tasks/self/_image/*",
|
|
40
|
+
"resources/daily/tasks/self/S*/*",
|
|
41
|
+
"resources/daily/tasks/self/S*/verify/*",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[tool.pytest.ini_options]
|
|
45
|
+
testpaths = ["tests"]
|
|
46
|
+
pythonpath = ["src"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""adapters: integration layer for agents under test (mini_swe_agent first)."""
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Local-only Codex exec adapter for the M4-C compatibility arm (D24)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import subprocess
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from traceverdict.adapters.mini_swe_agent import AdapterHarnessError, AdapterResult
|
|
13
|
+
from traceverdict.swebench_budget import ADAPTER_WALL_TIMEOUT_GRACE_S
|
|
14
|
+
|
|
15
|
+
PINNED_CODEX_VERSION = "0.144.4"
|
|
16
|
+
CODEX_TRAJECTORY_FORMAT = "codex-exec-jsonl-1"
|
|
17
|
+
CODEX_BINARY_IN_IMAGE = "/opt/traceverdict/codex"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _config_args(model_params: dict[str, Any]) -> list[str]:
|
|
21
|
+
"""Return the complete behavior-affecting Codex config overrides."""
|
|
22
|
+
effort = model_params.get("model_reasoning_effort")
|
|
23
|
+
if effort not in {"high", "medium"}:
|
|
24
|
+
raise AdapterHarnessError(
|
|
25
|
+
"model_reasoning_effort must be an explicitly approved M4-C value "
|
|
26
|
+
f"(high or medium), got {effort!r}"
|
|
27
|
+
)
|
|
28
|
+
expected = {
|
|
29
|
+
"auth_mode": "chatgpt_subscription",
|
|
30
|
+
"billing_mode": "subscription_unallocatable",
|
|
31
|
+
"approval_policy": "never",
|
|
32
|
+
"sandbox_mode": "workspace-write",
|
|
33
|
+
"web_search": "disabled",
|
|
34
|
+
"ephemeral": True,
|
|
35
|
+
"ignore_user_config": True,
|
|
36
|
+
"ignore_rules": True,
|
|
37
|
+
"concurrency": 1,
|
|
38
|
+
"network_mode": "bridge",
|
|
39
|
+
"container_cap_add": "SYS_ADMIN",
|
|
40
|
+
"container_seccomp": "unconfined",
|
|
41
|
+
"inner_sandbox": "workspace-write",
|
|
42
|
+
"auth_isolation": "per_tool_mount_namespace_hide_codex_home_v2",
|
|
43
|
+
}
|
|
44
|
+
mismatches = {
|
|
45
|
+
key: {"expected": value, "actual": model_params.get(key)}
|
|
46
|
+
for key, value in expected.items()
|
|
47
|
+
if model_params.get(key) != value
|
|
48
|
+
}
|
|
49
|
+
if mismatches:
|
|
50
|
+
raise AdapterHarnessError(
|
|
51
|
+
"immutable Codex M4-C identity mismatch: "
|
|
52
|
+
+ json.dumps(mismatches, sort_keys=True)
|
|
53
|
+
)
|
|
54
|
+
nested_expected = {
|
|
55
|
+
"sandbox_workspace_write": {"network_access": True},
|
|
56
|
+
"history": {"persistence": "none"},
|
|
57
|
+
"features": {
|
|
58
|
+
"shell_tool": True,
|
|
59
|
+
"shell_snapshot": False,
|
|
60
|
+
"unified_exec": False,
|
|
61
|
+
},
|
|
62
|
+
"hide_agent_reasoning": False,
|
|
63
|
+
}
|
|
64
|
+
nested_mismatches = {
|
|
65
|
+
key: {"expected": value, "actual": model_params.get(key)}
|
|
66
|
+
for key, value in nested_expected.items()
|
|
67
|
+
if model_params.get(key) != value
|
|
68
|
+
}
|
|
69
|
+
if nested_mismatches:
|
|
70
|
+
raise AdapterHarnessError(
|
|
71
|
+
"immutable Codex nested identity mismatch: "
|
|
72
|
+
+ json.dumps(nested_mismatches, sort_keys=True)
|
|
73
|
+
)
|
|
74
|
+
if model_params.get("service_tier") != "omitted":
|
|
75
|
+
raise AdapterHarnessError("service_tier must be explicitly omitted")
|
|
76
|
+
if model_params.get("temperature") != "omitted" or model_params.get("top_p") != "omitted":
|
|
77
|
+
raise AdapterHarnessError("temperature and top_p must be explicitly omitted")
|
|
78
|
+
return [
|
|
79
|
+
"-c", f"model_reasoning_effort=\"{effort}\"",
|
|
80
|
+
"-c", "approval_policy=\"never\"",
|
|
81
|
+
"-c", "web_search=\"disabled\"",
|
|
82
|
+
"-c", "history.persistence=\"none\"",
|
|
83
|
+
"-c", "hide_agent_reasoning=false",
|
|
84
|
+
"-c", "features.shell_tool=true",
|
|
85
|
+
"-c", "features.shell_snapshot=false",
|
|
86
|
+
"-c", "features.unified_exec=false",
|
|
87
|
+
"-c", "sandbox_workspace_write.network_access=true",
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def build_codex_command(
|
|
92
|
+
*, model_name: str, model_params: dict[str, Any], container_cwd: str
|
|
93
|
+
) -> list[str]:
|
|
94
|
+
return [
|
|
95
|
+
CODEX_BINARY_IN_IMAGE,
|
|
96
|
+
"exec",
|
|
97
|
+
"--json",
|
|
98
|
+
"--ephemeral",
|
|
99
|
+
"--ignore-user-config",
|
|
100
|
+
"--ignore-rules",
|
|
101
|
+
"--strict-config",
|
|
102
|
+
"--model",
|
|
103
|
+
model_name,
|
|
104
|
+
"--sandbox",
|
|
105
|
+
"workspace-write",
|
|
106
|
+
"-C",
|
|
107
|
+
container_cwd,
|
|
108
|
+
*_config_args(model_params),
|
|
109
|
+
"-",
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def parse_codex_jsonl(data: str) -> list[dict[str, Any]]:
|
|
114
|
+
records: list[dict[str, Any]] = []
|
|
115
|
+
for line_number, raw in enumerate(data.splitlines(), start=1):
|
|
116
|
+
if not raw.strip():
|
|
117
|
+
continue
|
|
118
|
+
try:
|
|
119
|
+
item = json.loads(raw)
|
|
120
|
+
except json.JSONDecodeError as exc:
|
|
121
|
+
raise AdapterHarnessError(
|
|
122
|
+
f"Codex JSONL line {line_number} is invalid: {exc}"
|
|
123
|
+
) from exc
|
|
124
|
+
if not isinstance(item, dict) or not isinstance(item.get("type"), str):
|
|
125
|
+
raise AdapterHarnessError(
|
|
126
|
+
f"Codex JSONL line {line_number} lacks an event type"
|
|
127
|
+
)
|
|
128
|
+
records.append(item)
|
|
129
|
+
if not records:
|
|
130
|
+
raise AdapterHarnessError("Codex produced no JSONL records")
|
|
131
|
+
return records
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def run_codex(
|
|
135
|
+
*,
|
|
136
|
+
instruction: str,
|
|
137
|
+
image: str,
|
|
138
|
+
docker_executable: str,
|
|
139
|
+
host_work_path: Path,
|
|
140
|
+
container_cwd: str,
|
|
141
|
+
model_name: str,
|
|
142
|
+
model_params: dict[str, Any],
|
|
143
|
+
litellm_model_registry: Path | None,
|
|
144
|
+
agent_version: str,
|
|
145
|
+
cost_limit: float,
|
|
146
|
+
step_limit: int,
|
|
147
|
+
wall_time_s: int,
|
|
148
|
+
work_dir: Path,
|
|
149
|
+
) -> AdapterResult:
|
|
150
|
+
"""Run Codex inside the local agent-layer image; never use a remote host."""
|
|
151
|
+
del litellm_model_registry, cost_limit, step_limit
|
|
152
|
+
if agent_version != PINNED_CODEX_VERSION:
|
|
153
|
+
raise AdapterHarnessError(
|
|
154
|
+
f"Codex version mismatch: expected {PINNED_CODEX_VERSION}, got {agent_version}"
|
|
155
|
+
)
|
|
156
|
+
auth_spec = os.environ.get("TRACEVERDICT_CODEX_AUTH_FILE")
|
|
157
|
+
if not auth_spec:
|
|
158
|
+
raise AdapterHarnessError("TRACEVERDICT_CODEX_AUTH_FILE is required locally")
|
|
159
|
+
auth_file = Path(auth_spec).resolve()
|
|
160
|
+
if not auth_file.is_file():
|
|
161
|
+
raise AdapterHarnessError("local Codex auth file does not exist")
|
|
162
|
+
work_dir.mkdir(parents=True, exist_ok=True)
|
|
163
|
+
jsonl_path = work_dir / "codex.events.jsonl"
|
|
164
|
+
stderr_path = work_dir / "codex.stderr.log"
|
|
165
|
+
command = build_codex_command(
|
|
166
|
+
model_name=model_name,
|
|
167
|
+
model_params=model_params,
|
|
168
|
+
container_cwd=container_cwd,
|
|
169
|
+
)
|
|
170
|
+
host_work = str(host_work_path.resolve()).replace("\\", "/")
|
|
171
|
+
host_auth = str(auth_file).replace("\\", "/")
|
|
172
|
+
docker_command = [
|
|
173
|
+
docker_executable,
|
|
174
|
+
"run",
|
|
175
|
+
"--rm",
|
|
176
|
+
"-i",
|
|
177
|
+
"--network",
|
|
178
|
+
"bridge",
|
|
179
|
+
"--cap-add",
|
|
180
|
+
"SYS_ADMIN",
|
|
181
|
+
"--security-opt",
|
|
182
|
+
"seccomp=unconfined",
|
|
183
|
+
"-e",
|
|
184
|
+
"CODEX_HOME=/run/traceverdict-codex",
|
|
185
|
+
"-v",
|
|
186
|
+
f"{host_work}:{container_cwd}",
|
|
187
|
+
"-v",
|
|
188
|
+
f"{host_auth}:/run/traceverdict-codex/auth.json",
|
|
189
|
+
"-w",
|
|
190
|
+
container_cwd,
|
|
191
|
+
image,
|
|
192
|
+
*command,
|
|
193
|
+
]
|
|
194
|
+
timeout = max(1, int(wall_time_s)) + ADAPTER_WALL_TIMEOUT_GRACE_S
|
|
195
|
+
try:
|
|
196
|
+
proc = subprocess.run(
|
|
197
|
+
docker_command,
|
|
198
|
+
input=instruction,
|
|
199
|
+
capture_output=True,
|
|
200
|
+
text=True,
|
|
201
|
+
encoding="utf-8",
|
|
202
|
+
errors="strict",
|
|
203
|
+
timeout=timeout,
|
|
204
|
+
check=False,
|
|
205
|
+
)
|
|
206
|
+
except subprocess.TimeoutExpired as exc:
|
|
207
|
+
raise AdapterHarnessError(
|
|
208
|
+
f"Codex exceeded adapter timeout {timeout}s", exit_reason="TimeExceeded"
|
|
209
|
+
) from exc
|
|
210
|
+
jsonl_path.write_text(proc.stdout or "", encoding="utf-8")
|
|
211
|
+
stderr_path.write_text(proc.stderr or "", encoding="utf-8")
|
|
212
|
+
records = parse_codex_jsonl(proc.stdout or "")
|
|
213
|
+
trajectory = {
|
|
214
|
+
"trajectory_format": CODEX_TRAJECTORY_FORMAT,
|
|
215
|
+
"records": records,
|
|
216
|
+
"jsonl_sha256": hashlib.sha256((proc.stdout or "").encode("utf-8")).hexdigest(),
|
|
217
|
+
"instruction_sha256": hashlib.sha256(instruction.encode("utf-8")).hexdigest(),
|
|
218
|
+
"info": {
|
|
219
|
+
"submission": "",
|
|
220
|
+
"model_stats": {"instance_cost": None},
|
|
221
|
+
"returncode": proc.returncode,
|
|
222
|
+
},
|
|
223
|
+
}
|
|
224
|
+
trajectory_path = work_dir / "codex.trajectory.json"
|
|
225
|
+
trajectory_path.write_text(
|
|
226
|
+
json.dumps(trajectory, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
227
|
+
)
|
|
228
|
+
return AdapterResult(
|
|
229
|
+
traj=trajectory,
|
|
230
|
+
traj_path=trajectory_path,
|
|
231
|
+
returncode=proc.returncode,
|
|
232
|
+
stdout=proc.stdout or "",
|
|
233
|
+
stderr=proc.stderr or "",
|
|
234
|
+
source_artifacts={"codex_jsonl": jsonl_path},
|
|
235
|
+
)
|