custos-code 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- custos_code/__init__.py +6 -0
- custos_code/adapters/__init__.py +194 -0
- custos_code/adapters/claude_code.py +266 -0
- custos_code/adapters/codex.py +437 -0
- custos_code/adapters/copilot.py +158 -0
- custos_code/adapters/devin.py +172 -0
- custos_code/adapters/machine.py +379 -0
- custos_code/adapters/otel.py +210 -0
- custos_code/adapters/state.py +164 -0
- custos_code/claims.py +319 -0
- custos_code/cli.py +789 -0
- custos_code/compress.py +113 -0
- custos_code/cost.py +216 -0
- custos_code/demo_fixtures/__init__.py +1 -0
- custos_code/demo_fixtures/ok_tests_0.jsonl +8 -0
- custos_code/demo_fixtures/trap_echo_0.jsonl +4 -0
- custos_code/demo_fixtures/trap_ghost_0.jsonl +4 -0
- custos_code/demo_fixtures/trap_piped_0.jsonl +4 -0
- custos_code/feedback.py +93 -0
- custos_code/hooks.py +648 -0
- custos_code/judge.py +338 -0
- custos_code/ledger.py +93 -0
- custos_code/models.py +129 -0
- custos_code/parsers.py +408 -0
- custos_code/report.py +317 -0
- custos_code/rerun.py +424 -0
- custos_code/review.py +381 -0
- custos_code/rules.py +464 -0
- custos_code/scope.py +471 -0
- custos_code/verdicts.py +296 -0
- custos_code-0.0.1.dist-info/METADATA +138 -0
- custos_code-0.0.1.dist-info/RECORD +35 -0
- custos_code-0.0.1.dist-info/WHEEL +4 -0
- custos_code-0.0.1.dist-info/entry_points.txt +2 -0
- custos_code-0.0.1.dist-info/licenses/LICENSE +21 -0
custos_code/verdicts.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Runs the ladder: rules -> (rerun) -> judge, and enforces the invariants at the boundary.
|
|
2
|
+
|
|
3
|
+
This is the only place that assembles VerdictRecords for a session. It asserts that no
|
|
4
|
+
judge-produced record is `contradicted`, that every `confirmed` carries evidence or a state
|
|
5
|
+
check, and that unsettled claims come back as `unwitnessed` rather than being dropped.
|
|
6
|
+
|
|
7
|
+
Escalation rule: a rule's verdict stands unless it is a low-confidence `unwitnessed` that the
|
|
8
|
+
rule itself marked as semantic (tier 4). Those, and claims with no rule at all, go to the judge,
|
|
9
|
+
which can only turn `unwitnessed` into `confirmed` — never the other way, and never into
|
|
10
|
+
`contradicted`. So adding a judge can only ever add evidence, not remove a deterministic finding.
|
|
11
|
+
|
|
12
|
+
Owner: Oliver.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import hashlib
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import subprocess
|
|
21
|
+
|
|
22
|
+
from . import claims as claims_mod
|
|
23
|
+
from . import parsers
|
|
24
|
+
from . import rerun as rerun_mod
|
|
25
|
+
from .judge import Backend, window_for_all
|
|
26
|
+
from .models import Claim, ClaimType, EventKind, LedgerEvent, Verdict, VerdictRecord
|
|
27
|
+
from .rules import check as rules_check
|
|
28
|
+
|
|
29
|
+
# Claim types that only a tool log can witness. Edits, commits and deletions survive in state, so
|
|
30
|
+
# they stay settleable on a class-R record; these do not.
|
|
31
|
+
NEEDS_TOOL_LOG = frozenset(
|
|
32
|
+
{
|
|
33
|
+
ClaimType.RUN_CMD,
|
|
34
|
+
ClaimType.RUN_TESTS,
|
|
35
|
+
ClaimType.BUILD,
|
|
36
|
+
ClaimType.DEPLOY,
|
|
37
|
+
ClaimType.READ,
|
|
38
|
+
ClaimType.OBSERVED_OUTPUT,
|
|
39
|
+
ClaimType.VERIFY,
|
|
40
|
+
}
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def known_incomplete(ledger: list[LedgerEvent]) -> str | None:
|
|
45
|
+
"""The adapter's own statement that this record has no tool log (see adapters/state.py)."""
|
|
46
|
+
for event in ledger:
|
|
47
|
+
if event.kind is EventKind.META and (event.input or {}).get("event") == "no_tool_log":
|
|
48
|
+
return str((event.input or {}).get("note") or "the record has no tool log")
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _enforce(rec: VerdictRecord) -> VerdictRecord:
|
|
53
|
+
if rec.method == "judge" and rec.verdict == Verdict.CONTRADICTED:
|
|
54
|
+
raise AssertionError("invariant 3: the judge cannot emit contradicted")
|
|
55
|
+
if (
|
|
56
|
+
rec.verdict == Verdict.CONFIRMED
|
|
57
|
+
and not rec.evidence
|
|
58
|
+
and rec.method not in ("state", "rerun")
|
|
59
|
+
):
|
|
60
|
+
raise AssertionError("confirmed without evidence or a state check")
|
|
61
|
+
return rec
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _escalates(rec: VerdictRecord | None) -> bool:
|
|
65
|
+
"""True when the judge could add something a rule could not."""
|
|
66
|
+
if rec is None:
|
|
67
|
+
return True
|
|
68
|
+
return rec.verdict == Verdict.UNWITNESSED and rec.tier >= 4
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def apply_reruns(
|
|
72
|
+
claims: list[Claim], records: list[VerdictRecord], ledger: list[LedgerEvent],
|
|
73
|
+
) -> list[VerdictRecord]:
|
|
74
|
+
"""Settle open test claims from claim-bound, numbered harness reruns.
|
|
75
|
+
|
|
76
|
+
Launching remains in the Stop hook so read-only audits never execute repository code.
|
|
77
|
+
A result cannot stand in for a different claim/session or survive later tool activity.
|
|
78
|
+
Legacy events without a launch boundary remain available to the judge, not this rule.
|
|
79
|
+
"""
|
|
80
|
+
settled = []
|
|
81
|
+
for claim, rec in zip(claims, records, strict=True):
|
|
82
|
+
ctype = claim.type
|
|
83
|
+
if ctype == ClaimType.OTHER:
|
|
84
|
+
ctype = claims_mod.classify(claim.text) or ClaimType.OTHER
|
|
85
|
+
if ctype not in _RERUNNABLE or claim.polarity != "did" or rec.verdict not in (
|
|
86
|
+
Verdict.UNWITNESSED, Verdict.UNRECORDED,
|
|
87
|
+
):
|
|
88
|
+
settled.append(rec)
|
|
89
|
+
continue
|
|
90
|
+
candidates = []
|
|
91
|
+
for event in ledger:
|
|
92
|
+
meta = event.input or {}
|
|
93
|
+
boundary = meta.get("report_seq")
|
|
94
|
+
if (event.kind != EventKind.RERUN or event.tool != "rerun_tests"
|
|
95
|
+
or event.session_id != claim.session_id or event.flags.sidechain
|
|
96
|
+
or meta.get("claim_id") != claim.id or meta.get("claim_text") != claim.text
|
|
97
|
+
or meta.get("claim_kind", "run_tests") != ctype.value
|
|
98
|
+
or not isinstance(boundary, int)
|
|
99
|
+
or event.seq <= boundary):
|
|
100
|
+
continue
|
|
101
|
+
if any(e.session_id == claim.session_id and not e.flags.sidechain
|
|
102
|
+
and e.kind in (EventKind.CALL, EventKind.RESULT) and e.seq > boundary
|
|
103
|
+
for e in ledger):
|
|
104
|
+
continue
|
|
105
|
+
candidates.append(event)
|
|
106
|
+
if not candidates:
|
|
107
|
+
settled.append(rec)
|
|
108
|
+
continue
|
|
109
|
+
event = max(candidates, key=lambda e: e.seq)
|
|
110
|
+
parsed = parsers.parse(event.output or "", event.exit_code)
|
|
111
|
+
verdict = Verdict.UNRECORDED
|
|
112
|
+
why = "Tier 3 re-run has no complete, recognised test outcome."
|
|
113
|
+
qualifier = None
|
|
114
|
+
flags = event.flags
|
|
115
|
+
if not (flags.timed_out or flags.interrupted or flags.truncated or flags.piped):
|
|
116
|
+
if ctype == ClaimType.BUILD:
|
|
117
|
+
if event.exit_code == 0 and not flags.error:
|
|
118
|
+
verdict = Verdict.CONFIRMED
|
|
119
|
+
why = "Tier 3 build completed successfully."
|
|
120
|
+
elif event.exit_code not in (None, 0, 126, 127):
|
|
121
|
+
verdict = Verdict.CONTRADICTED
|
|
122
|
+
why = f"Tier 3 build failed (exit {event.exit_code})."
|
|
123
|
+
elif parsed is not None and event.exit_code is not None:
|
|
124
|
+
if parsed.failed or parsed.errors or parsed.collected == 0:
|
|
125
|
+
verdict = Verdict.CONTRADICTED
|
|
126
|
+
why = (f"Tier 3 {parsed.runner}: {parsed.passed} passed, "
|
|
127
|
+
f"{parsed.failed} failed, {parsed.errors} errors"
|
|
128
|
+
+ ("; no tests collected." if parsed.collected == 0 else "."))
|
|
129
|
+
elif event.exit_code == 0 and not flags.error and parsed.passed > 0:
|
|
130
|
+
verdict = Verdict.CONFIRMED
|
|
131
|
+
why = f"Tier 3 {parsed.runner}: {parsed.passed} passed, 0 failed."
|
|
132
|
+
counts = [o for o in claim.objects if o.isdigit()]
|
|
133
|
+
if counts and any(int(n) != parsed.passed for n in counts):
|
|
134
|
+
verdict = Verdict.QUALIFIED
|
|
135
|
+
qualifier = f"{', '.join(counts)} claimed, {parsed.passed} passed"
|
|
136
|
+
settled.append(_enforce(VerdictRecord(
|
|
137
|
+
claim_id=claim.id, verdict=verdict, tier=3, method="rerun",
|
|
138
|
+
confidence=0.9, evidence=[event.seq], rationale=why, qualifier=qualifier,
|
|
139
|
+
)))
|
|
140
|
+
return settled
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def run(
|
|
144
|
+
claims: list[Claim],
|
|
145
|
+
ledger: list[LedgerEvent],
|
|
146
|
+
repo_root: str | None,
|
|
147
|
+
backend: Backend | None = None,
|
|
148
|
+
) -> list[VerdictRecord]:
|
|
149
|
+
settled: dict[str, VerdictRecord] = {}
|
|
150
|
+
pending: list[Claim] = []
|
|
151
|
+
for claim in claims:
|
|
152
|
+
found = rules_check(claim, ledger, repo_root)
|
|
153
|
+
if _escalates(found):
|
|
154
|
+
pending.append(claim)
|
|
155
|
+
rec = (
|
|
156
|
+
found
|
|
157
|
+
if found is not None
|
|
158
|
+
else VerdictRecord(
|
|
159
|
+
claim_id=claim.id,
|
|
160
|
+
verdict=Verdict.UNWITNESSED,
|
|
161
|
+
tier=4,
|
|
162
|
+
method="rule",
|
|
163
|
+
confidence=0.5,
|
|
164
|
+
evidence=[],
|
|
165
|
+
rationale="No deterministic rule applies to this claim type.",
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
settled[claim.id] = _enforce(rec)
|
|
169
|
+
|
|
170
|
+
if backend is not None and pending:
|
|
171
|
+
win = window_for_all(ledger, pending)
|
|
172
|
+
for jrec in backend.judge(pending, win):
|
|
173
|
+
if jrec.verdict == Verdict.CONFIRMED: # the judge may only upgrade unwitnessed
|
|
174
|
+
settled[jrec.claim_id] = _enforce(jrec)
|
|
175
|
+
else:
|
|
176
|
+
prev = settled[jrec.claim_id]
|
|
177
|
+
prev.rationale = jrec.rationale or prev.rationale
|
|
178
|
+
prev.method = "judge"
|
|
179
|
+
prev.tier = 4
|
|
180
|
+
|
|
181
|
+
note = known_incomplete(ledger)
|
|
182
|
+
if note:
|
|
183
|
+
for claim in claims:
|
|
184
|
+
rec = settled[claim.id]
|
|
185
|
+
if rec.verdict is Verdict.UNWITNESSED and claim.type in NEEDS_TOOL_LOG:
|
|
186
|
+
rec.verdict = Verdict.UNRECORDED
|
|
187
|
+
rec.rationale = f"{note}; this claim needs one."
|
|
188
|
+
return apply_reruns(claims, [settled[c.id] for c in claims], ledger)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def summary(records: list[VerdictRecord]) -> dict[str, int]:
|
|
192
|
+
s = {v.value: 0 for v in Verdict}
|
|
193
|
+
for r in records:
|
|
194
|
+
s[r.verdict.value] += 1
|
|
195
|
+
return s
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
# ---------------------------------------------------------------------------------------------
|
|
199
|
+
# Tier 3 gating: when re-execution is worth launching at all.
|
|
200
|
+
#
|
|
201
|
+
# `rerun.py` (Anush's) knows HOW to re-run safely -- committed config only, read from the git
|
|
202
|
+
# object store so the agent cannot steer it, in a worktree, with a timeout, async off the Stop
|
|
203
|
+
# hook's critical path. Nothing decided WHEN, so nothing ever called it.
|
|
204
|
+
#
|
|
205
|
+
# The failure mode to design against is not a bad re-run, it is a checker that spends a minute
|
|
206
|
+
# re-running things that settle nothing. So the gate is deliberately narrow: every condition below
|
|
207
|
+
# must hold, and the common case is that none of them do.
|
|
208
|
+
# ---------------------------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
RERUN_BUDGET_PER_SESSION = 2
|
|
211
|
+
|
|
212
|
+
# Claim types a re-execution can actually settle. Re-running proves a suite passes; it cannot
|
|
213
|
+
# prove a file was edited, a commit was made, or a page was read.
|
|
214
|
+
_RERUNNABLE = frozenset({ClaimType.RUN_TESTS, ClaimType.BUILD})
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _tree_key(repo_root: str) -> str | None:
|
|
218
|
+
"""A fingerprint of the tree a re-run would execute against.
|
|
219
|
+
|
|
220
|
+
Re-running the same commit with the same working tree gives the same answer, so the second
|
|
221
|
+
attempt is pure cost. HEAD alone is not enough -- uncommitted edits are part of what the
|
|
222
|
+
report is about (rerun.py's E3) -- so the porcelain status goes in too.
|
|
223
|
+
"""
|
|
224
|
+
try:
|
|
225
|
+
head = subprocess.run(["git", "-C", repo_root, "rev-parse", "HEAD"],
|
|
226
|
+
capture_output=True, text=True, timeout=5)
|
|
227
|
+
status = subprocess.run(["git", "-C", repo_root, "status", "--porcelain"],
|
|
228
|
+
capture_output=True, text=True, timeout=10)
|
|
229
|
+
except (OSError, subprocess.SubprocessError):
|
|
230
|
+
return None
|
|
231
|
+
if head.returncode != 0:
|
|
232
|
+
return None
|
|
233
|
+
return hashlib.sha256((head.stdout + status.stdout).encode()).hexdigest()[:16]
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def should_rerun(claim: Claim, rec: VerdictRecord, repo_root: str | None,
|
|
237
|
+
already: set[str] | None = None, budget: int = RERUN_BUDGET_PER_SESSION) -> bool:
|
|
238
|
+
"""True when re-executing could change this verdict and has not been tried on this tree.
|
|
239
|
+
|
|
240
|
+
Every condition has to hold. In order, and each exists because of a way this wastes time:
|
|
241
|
+
|
|
242
|
+
1. The verdict is still open. A `confirmed` or `contradicted` is settled on evidence; re-running
|
|
243
|
+
cannot improve it and a disagreement would be a second opinion, not a second source.
|
|
244
|
+
2. The claim is about running something. Tier 3 answers "does it pass", nothing else.
|
|
245
|
+
3. There is a repo. No git work tree means no worktree to materialise, so `rerun_tests` would
|
|
246
|
+
raise -- which is the bug PR #51's review found in the eval harness.
|
|
247
|
+
4. There is a committed runner config. Auto-detection reads HEAD, so a repo with no test
|
|
248
|
+
configuration has nothing to run and would fail for reasons unrelated to the claim.
|
|
249
|
+
5. Budget, and not already tried on this exact tree. Same commit plus same working tree gives
|
|
250
|
+
the same answer, so repeating it is pure latency.
|
|
251
|
+
"""
|
|
252
|
+
if claim.polarity != "did" or rec.verdict not in (Verdict.UNWITNESSED, Verdict.UNRECORDED):
|
|
253
|
+
return False
|
|
254
|
+
# `review.py` -- the path that actually ships -- labels every claim `OTHER`, because its one
|
|
255
|
+
# call extracts and judges but does not classify. Keying the gate on the type alone meant it
|
|
256
|
+
# could only ever fire on the superseded ladder, i.e. never. Fall back to the deterministic
|
|
257
|
+
# text classifier in claims.py, which is what the ladder uses anyway.
|
|
258
|
+
ctype = claim.type
|
|
259
|
+
if ctype in (ClaimType.OTHER, None):
|
|
260
|
+
ctype = claims_mod.classify(claim.text) or ClaimType.OTHER
|
|
261
|
+
if ctype not in _RERUNNABLE:
|
|
262
|
+
return False
|
|
263
|
+
if not repo_root or not os.path.isdir(repo_root):
|
|
264
|
+
return False
|
|
265
|
+
if not os.path.isdir(os.path.join(repo_root, ".git")) and \
|
|
266
|
+
not os.path.isfile(os.path.join(repo_root, ".git")):
|
|
267
|
+
return False
|
|
268
|
+
if rerun_command(claim, repo_root) is None:
|
|
269
|
+
return False
|
|
270
|
+
seen = already if already is not None else set()
|
|
271
|
+
if len(seen) >= budget:
|
|
272
|
+
return False
|
|
273
|
+
key = _tree_key(repo_root)
|
|
274
|
+
return key is not None and f"{claim.id}:{key}" not in seen
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def rerun_key(claim: Claim, repo_root: str) -> str | None:
|
|
278
|
+
"""The dedupe key `should_rerun` checks, for a caller to record after launching."""
|
|
279
|
+
key = _tree_key(repo_root)
|
|
280
|
+
return f"{claim.id}:{key}" if key else None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def rerun_kind(claim: Claim) -> ClaimType:
|
|
284
|
+
return (claims_mod.classify(claim.text) or ClaimType.OTHER
|
|
285
|
+
if claim.type == ClaimType.OTHER else claim.type)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def rerun_command(claim: Claim, repo_root: str) -> list[str] | None:
|
|
289
|
+
"""Choose a committed command only for the check the claim actually describes."""
|
|
290
|
+
ctype = rerun_kind(claim)
|
|
291
|
+
if ctype not in _RERUNNABLE:
|
|
292
|
+
return None
|
|
293
|
+
# BUILD also includes lint/typechecking; a successful build cannot verify those.
|
|
294
|
+
if ctype == ClaimType.BUILD and not re.search(r"\b(?:build|built|compil\w*)\b", claim.text, re.I):
|
|
295
|
+
return None
|
|
296
|
+
return rerun_mod.detect_command(repo_root, ctype.value)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: custos-code
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Checks a coding agent's final report against the log of what it actually did.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: agents,claude-code,hooks,observability,verification
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
12
|
+
Requires-Python: >=3.12
|
|
13
|
+
Requires-Dist: anthropic>=0.40
|
|
14
|
+
Requires-Dist: duckdb>=1.0
|
|
15
|
+
Requires-Dist: openai>=1.40
|
|
16
|
+
Requires-Dist: pydantic>=2.7
|
|
17
|
+
Requires-Dist: rich>=13.7
|
|
18
|
+
Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
19
|
+
Requires-Dist: typer>=0.12
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: hypothesis>=6; extra == 'dev'
|
|
22
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
24
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# Custos Code
|
|
28
|
+
|
|
29
|
+
Checks a coding agent's final report against the log of what it actually did.
|
|
30
|
+
|
|
31
|
+
An agent finishes and says "implemented the feature, ran the tests, all passing." Custos Code reads the harness-written action log, splits the report into claims, and marks each one **confirmed**, **contradicted**, **unwitnessed**, **unrecorded**, or **qualified**, with the ledger lines that back the verdict. Contradictions go back to the agent before it is allowed to stop.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
custos-code session 4f2a… · 63 events
|
|
35
|
+
✓ confirmed edited auth/middleware.py tier 1 · #14 Edit, git diff agrees
|
|
36
|
+
✓ confirmed added tests/test_rate_limit.py tier 1 · #31 Write, file present
|
|
37
|
+
✗ contradicted ran the suite, all 12 passing tier 2 · #41 `pytest | tail -5` exit 0, "collected 0 items"
|
|
38
|
+
? unwitnessed ready to merge no CI, no git status after #41
|
|
39
|
+
stop blocked · 1 contradicted · evidence returned to agent
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Status
|
|
43
|
+
|
|
44
|
+
Pre-build. The design, research, plan, and evidence protocol are in `docs/`. Start with [AGENTS.md](AGENTS.md).
|
|
45
|
+
|
|
46
|
+
- [docs/DESIGN.md](docs/DESIGN.md) — problem, customers, verification ladder, feasibility, product sketches, benchmark, system design, prize strategy, adversarial review
|
|
47
|
+
- [docs/RESEARCH.md](docs/RESEARCH.md) — the evidence: prevalence with denominators, cost, current workarounds, tool landscape, 40 seed cases
|
|
48
|
+
- [docs/PLAN.md](docs/PLAN.md) — who owns what, phases, parallel tracks
|
|
49
|
+
- [docs/EVIDENCE_PLAN.md](docs/EVIDENCE_PLAN.md) — pre-registered study: accuracy, time saved, retention, usability
|
|
50
|
+
- [docs/OPEN_QUESTIONS.md](docs/OPEN_QUESTIONS.md) — every unresolved decision with an owner
|
|
51
|
+
|
|
52
|
+
## See it work
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
uv sync
|
|
56
|
+
export OPENAI_API_KEY=...
|
|
57
|
+
uv run custos-code demo # the whole loop on a known trap, live
|
|
58
|
+
uv run custos-code demo --scenario honest # the control: nothing blocks
|
|
59
|
+
uv run custos-code check --last # your own most recent session
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`demo` prints five things from the fixture's own tool log: what was asked, what the agent actually
|
|
63
|
+
did, what it said, the receipt, and the deterministic nudge that goes back. `--format html --out
|
|
64
|
+
card.html` writes a self-contained report card; `--format markdown` writes what the PR bot posts.
|
|
65
|
+
|
|
66
|
+
## Prototype
|
|
67
|
+
|
|
68
|
+
`docs/prototype/index.html` is an interactive, non-functional mock of the editor experience: marks on the agent's message, the evidence panel, editor decorations, the auto-mode loop, and the PR receipt. It is a single self-contained file:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
open docs/prototype/index.html # macOS
|
|
72
|
+
# or: python3 -m http.server -d docs/prototype 8765 → http://localhost:8765
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Local setup
|
|
76
|
+
|
|
77
|
+
Install [uv](https://docs.astral.sh/uv/getting-started/installation/) once. On macOS
|
|
78
|
+
with Homebrew: `brew install uv`. From the repository directory:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
uv python install
|
|
82
|
+
make sync
|
|
83
|
+
make check
|
|
84
|
+
uv run custos-code check --last
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`.python-version` selects Python 3.12, independently of your shell's pyenv or
|
|
88
|
+
Conda default. `make sync` installs the project and developer tools from the
|
|
89
|
+
committed `uv.lock`; it fails if the lockfile needs updating. After an intentional
|
|
90
|
+
dependency change, run `uv lock` and include the lockfile in the same PR.
|
|
91
|
+
|
|
92
|
+
`make build` produces a wheel and source distribution in `dist/`. CI runs the
|
|
93
|
+
same checks, installs both distributions in clean environments, and checks the
|
|
94
|
+
installed CLI outside the source checkout. CI uses locked installs following
|
|
95
|
+
the [uv integration guide](https://docs.astral.sh/uv/guides/integration/github/).
|
|
96
|
+
|
|
97
|
+
## Bench container
|
|
98
|
+
|
|
99
|
+
With Docker installed and running, build from the repository root:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
docker build -t custos-code-bench .
|
|
103
|
+
docker run --rm --network none custos-code-bench
|
|
104
|
+
docker run --rm --network none custos-code-bench python -m pytest --version
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The image contains Python 3.12, uv 0.12.17, git, the installed Custos Code package,
|
|
108
|
+
developer dependencies, and scenario descriptions under `/app/bench/scenarios`.
|
|
109
|
+
It runs as a non-root user in writable `/workspace`; the default command shows
|
|
110
|
+
CLI help. The benchmark orchestration and fixture repos are not implemented
|
|
111
|
+
yet, so this is their execution environment, not a working benchmark command.
|
|
112
|
+
Agent CLIs and their credentials are not installed.
|
|
113
|
+
|
|
114
|
+
The Docker build context is an allowlist that excludes local session logs,
|
|
115
|
+
credentials, caches, and git history. For a local Python fixture, mount only
|
|
116
|
+
that fixture (including its git metadata when state checks need it):
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
docker run --rm --network none \
|
|
120
|
+
--mount type=bind,src="$(pwd)/path/to/fixture",dst=/workspace,readonly \
|
|
121
|
+
custos-code-bench python -m pytest -p no:cacheprovider
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
This read-only example suits tests that do not write into the fixture. Agent
|
|
125
|
+
bench runs will need a disposable writable checkout and explicit network and
|
|
126
|
+
credential configuration. Non-Python runners require additional toolchains.
|
|
127
|
+
|
|
128
|
+
## Why
|
|
129
|
+
|
|
130
|
+
Across 20,574 real coding-agent sessions, 22.58% of 16,118 validated misalignment episodes were the agent misreporting its own work, and only 2.99% of resolved episodes were self-corrected. Every agent vendor attaches an action log; none checks the report against it. Sources and denominators: `docs/RESEARCH.md`.
|
|
131
|
+
|
|
132
|
+
## Working in this repo
|
|
133
|
+
|
|
134
|
+
Read `AGENTS.md`. Flag anything undecided with `NEEDS-DECISION(owner):`. Local by default; secrets are redacted at ingest; fixtures are synthetic.
|
|
135
|
+
|
|
136
|
+
## License
|
|
137
|
+
|
|
138
|
+
MIT (see `LICENSE`).
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
custos_code/__init__.py,sha256=Bn27eigtdi0-EG278tO87Z3tYvpy9l9-jsFVrHg-wJA,266
|
|
2
|
+
custos_code/claims.py,sha256=wStiEn02kljl0OwlJvySKe3B6LCr83R7xoVjkurdVsI,16592
|
|
3
|
+
custos_code/cli.py,sha256=gRKeIExpUIhwqoRyT1JrK6psXNudvRmn4BOd7nBZk-8,34849
|
|
4
|
+
custos_code/compress.py,sha256=UDJMgcxmfpV-dtruDk8uD9Ye0YbTHIARgMywDwCx3nM,5329
|
|
5
|
+
custos_code/cost.py,sha256=FkvJBBoBpiFS3vFlvCBvXv-U7mKax2BgGo-XNEbWJQA,9331
|
|
6
|
+
custos_code/feedback.py,sha256=9Z7bvBZCXcSrWOmYit6VrojebffNv6X_sEIjPS0e4po,4992
|
|
7
|
+
custos_code/hooks.py,sha256=5I6fLj0s2wLbt4b7Uu_lyO-kVXpK49VckneelVv_oqw,32859
|
|
8
|
+
custos_code/judge.py,sha256=7GSXtpHUX7f2TI2LT4slD_G0UquSeehkVJFTiUcx6Xk,15351
|
|
9
|
+
custos_code/ledger.py,sha256=gGv1TvLb13fWXm1rytf9GqMm9kGEaWzVzTRmwfu1YhI,3410
|
|
10
|
+
custos_code/models.py,sha256=8NsYsnYITr6Bn6RDDXMmv-4xBELSkOJwJFu2ssxPxyo,4440
|
|
11
|
+
custos_code/parsers.py,sha256=O_ZVOJR0jyKs_PI2vdRMoJOPWfZV1x9B5AJGgtRUTlM,16853
|
|
12
|
+
custos_code/report.py,sha256=L7NZqdsKRSV5DxN0jr52lxQhE9eXxzSgwuJualiTVNg,12832
|
|
13
|
+
custos_code/rerun.py,sha256=OjNdL8g6QjkCn3xlh-G35y71ATipjkTngCqqLj729z8,17540
|
|
14
|
+
custos_code/review.py,sha256=UMIUQYZz5xuzDa2x8S49wogtNKTOumdUsVD9cd6QvA8,22129
|
|
15
|
+
custos_code/rules.py,sha256=DlDNESBOtt-PvUV5LpKSzhggS5lQHmFPhMCkYdc6qIc,26897
|
|
16
|
+
custos_code/scope.py,sha256=SxfDAyqAGyx7HGlvRPMacpUH-V1IA22l-_pXhiDtdi0,21897
|
|
17
|
+
custos_code/verdicts.py,sha256=ojg7L1XZsFMcXpMwnxrrwS8Mgjnwb17pgQLftlY6_xU,13355
|
|
18
|
+
custos_code/adapters/__init__.py,sha256=tIWo9VgbaZc5VECTWHt3ruh__-V0AIpE1hdoRMhdokw,7651
|
|
19
|
+
custos_code/adapters/claude_code.py,sha256=Pc0puiATaT00pOqGBcA7bMvZPfXrDXcBPDF2Fadmfac,12170
|
|
20
|
+
custos_code/adapters/codex.py,sha256=sDVoD9klNl1zkKKEubiaOk2EirZWg7IF3EWN0LLlJRg,16712
|
|
21
|
+
custos_code/adapters/copilot.py,sha256=TScDhVtLJi39APqrl23iIEZVx0SlcMTaojt-_yyc_PI,6682
|
|
22
|
+
custos_code/adapters/devin.py,sha256=UUxG-JSJj6FlSmhZ9jY5S89O7HqPHAGbmFlzT6RKH2g,7013
|
|
23
|
+
custos_code/adapters/machine.py,sha256=cE_etUmC2XZmUqXj3neHmumJmd5f9HwqPJB9QyFlNl8,18088
|
|
24
|
+
custos_code/adapters/otel.py,sha256=S6PcLbvGMPGTL0zCZeNAvXK21k10Dhje2jd0XnIQJw0,8509
|
|
25
|
+
custos_code/adapters/state.py,sha256=D8ifHC_0YwQmbh8qNNBIJR2ZN--j9RTQqOKt8Nvvi6M,6051
|
|
26
|
+
custos_code/demo_fixtures/__init__.py,sha256=D4d4sqOtaMTvkQO40nGzDAIPTzd1f0EgPDzNxOmktsA,58
|
|
27
|
+
custos_code/demo_fixtures/ok_tests_0.jsonl,sha256=vvhUaqu9lgNpR36CqTQdb0NVNRHW4X0sddbdspUCyK8,3804
|
|
28
|
+
custos_code/demo_fixtures/trap_echo_0.jsonl,sha256=27WDc8k5TQ0dYCN2fo_XFLHwEsgl399MpTs92jlMZNE,1470
|
|
29
|
+
custos_code/demo_fixtures/trap_ghost_0.jsonl,sha256=1y6L6cK9jqhkE36KP8NIOV7eeh4kM9GqMddDAoZBrFA,1468
|
|
30
|
+
custos_code/demo_fixtures/trap_piped_0.jsonl,sha256=Iw847yYgu2C5EHw95lHD4bOJb-gwf9IxjaiwHq84PCk,1546
|
|
31
|
+
custos_code-0.0.1.dist-info/METADATA,sha256=5NmHGRdJghadiT-MFIYmPMl5qw6rGNqenUVR6nEbuNI,6358
|
|
32
|
+
custos_code-0.0.1.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
|
|
33
|
+
custos_code-0.0.1.dist-info/entry_points.txt,sha256=OWy1MAzQ6WUrGJxAZhg0KPxj1F4mmxuersbyxRk0tO4,52
|
|
34
|
+
custos_code-0.0.1.dist-info/licenses/LICENSE,sha256=iFXb23ZCCdM4ggFRq6IFLF_yWPG8GyCeGzYh0mFPxqM,1069
|
|
35
|
+
custos_code-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Oliver Zhang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|