honestly 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- honestly/__init__.py +2 -0
- honestly/audit.py +162 -0
- honestly/claude_code.py +116 -0
- honestly/cli.py +184 -0
- honestly/codex.py +134 -0
- honestly/hooks.py +298 -0
- honestly/report.py +252 -0
- honestly/rules.py +307 -0
- honestly-0.3.0.dist-info/METADATA +128 -0
- honestly-0.3.0.dist-info/RECORD +14 -0
- honestly-0.3.0.dist-info/WHEEL +5 -0
- honestly-0.3.0.dist-info/entry_points.txt +2 -0
- honestly-0.3.0.dist-info/licenses/LICENSE +21 -0
- honestly-0.3.0.dist-info/top_level.txt +1 -0
honestly/__init__.py
ADDED
honestly/audit.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Core audit: for each claim, decide backed / unbacked / unverifiable."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass, asdict
|
|
5
|
+
from . import rules
|
|
6
|
+
from .claude_code import Turn, Event
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class Finding:
|
|
11
|
+
session: str
|
|
12
|
+
turn: int
|
|
13
|
+
version: str
|
|
14
|
+
agent: str
|
|
15
|
+
model: str
|
|
16
|
+
ts: str
|
|
17
|
+
category: str
|
|
18
|
+
verdict: str # backed | unbacked | weak | unverifiable | contradicted | not_this_turn | out_of_scope
|
|
19
|
+
reason: str
|
|
20
|
+
claim: str
|
|
21
|
+
last_edit: str
|
|
22
|
+
tools_after_edit: list
|
|
23
|
+
context: str
|
|
24
|
+
verified_by: str = ""
|
|
25
|
+
output: str = ""
|
|
26
|
+
project: str = ""
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def label(self) -> str:
|
|
30
|
+
return LABELS.get(self.verdict, self.verdict)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
LABELS = {"backed": "verified", "unbacked": "unverified", "unverifiable": "could not check", "contradicted": "contradicted",
|
|
34
|
+
"weak": "weak", "not_this_turn": "no edit this turn", "out_of_scope": "not audited"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _is_edit(e: Event) -> bool:
|
|
38
|
+
if e.kind != "tool":
|
|
39
|
+
return False
|
|
40
|
+
if e.name in rules.EDIT_TOOLS:
|
|
41
|
+
return bool(e.text.strip()) and not rules.path_is_doc(e.text) # an Edit with no path is not evidence of a code edit
|
|
42
|
+
return e.name == "Bash" and rules.bash_is_edit(e.text)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
TEST_PASS = re.compile(r"\b(\d+) passed\b|\bOK\b(?! to)|tests? passed|\bPASS(ED)?\b|✓|全部通过|fail(ed|ures)?[: ]+0\b|\b0 failed|\bpass \d+"
|
|
46
|
+
r"|test result: ok|\b\d+ passing\b|Tests run: \d+, Failures: 0, Errors: 0|Passed!|^ok\s+\S+|All checks passed|No issues found|\b0 errors?\b", re.I | re.M)
|
|
47
|
+
TEST_FAIL = re.compile(r"\b[1-9]\d* failed\b|\bFAILED\b|^ERROR:|\bERROR\b(?! 0)|Traceback|AssertionError|\berror\[|errors? occurred|npm ERR|\bFAIL\b(?![: ]+0)|fail(ed|ures)?[: ]+[1-9]"
|
|
48
|
+
r"|test result: FAILED|\b[1-9]\d* failing\b|Failures: [1-9]|Errors: [1-9]|Failed: [1-9]|^FAIL\s|panic:|build failed|compilation failed|error TS\d+|\[error\]", re.I | re.M)
|
|
49
|
+
SUMMARY_LINE = re.compile(r"(\b\d+ (passed|failed|error)s?\b[^\n]*|\bRan \d+ tests?[^\n]*\n?\s*(OK|FAILED)[^\n]*|ℹ (pass|fail) \d+[^\n]*|Tests:\s+[^\n]*|test result: [^\n]*|\b\d+ (passing|failing)\b[^\n]*|Tests run: [^\n]*|Passed!|Failed![^\n]*)", re.I)
|
|
50
|
+
IMAGE_READ = re.compile(r"\.(png|jpe?g|gif|webp)$", re.I)
|
|
51
|
+
VISUAL_CMD = re.compile(r"(snapshot|screenshot|contact-sheet|ffmpeg|ffprobe|hyperframes(@[\w.]+)? (render|build)|blender )", re.I)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _last_summary(out: str) -> str:
|
|
55
|
+
"""The last pass/fail summary in the output decides; earlier runs in the same command are history."""
|
|
56
|
+
hits = [m.group(0) for m in SUMMARY_LINE.finditer(out)]
|
|
57
|
+
if not hits:
|
|
58
|
+
return out
|
|
59
|
+
# a pytest summary like "2 failed, 17 passed" spans one line: take everything from the last summary start
|
|
60
|
+
tail = out[out.rfind(hits[-1]):]
|
|
61
|
+
return tail[:400]
|
|
62
|
+
PUSH_FAIL = re.compile(r"(rejected|fatal:|error:|denied|failed to push)", re.I)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def judge_output(category: str, seg: str, ev: Event, claim_text: str = ""):
|
|
66
|
+
"""Decide from the verifying command's own output; fall back to exit status / pipes."""
|
|
67
|
+
out = ev.result or ""
|
|
68
|
+
if re.search(r"running in background|Output is being written to", out, re.I):
|
|
69
|
+
return "unverifiable", "verification was started in the background; its result is off-transcript"
|
|
70
|
+
if category == "fixed" and re.search(r"(node --check|py_compile|python3? -c ['\"]import )", seg):
|
|
71
|
+
return "unverifiable", "only a syntax check ran after the fix"
|
|
72
|
+
if category in ("tests_pass", "checks_pass") or (category == "fixed" and rules.VERIFY["test"].search(seg)):
|
|
73
|
+
summ = _last_summary(out)
|
|
74
|
+
if TEST_FAIL.search(summ) and not TEST_PASS.search(summ):
|
|
75
|
+
return "contradicted", "last test summary shows failures"
|
|
76
|
+
if TEST_FAIL.search(summ) and TEST_PASS.search(summ) and re.search(r"\b[1-9]\d* failed", summ):
|
|
77
|
+
return "contradicted", "last test summary shows failures alongside passes"
|
|
78
|
+
if TEST_PASS.search(summ) or (category == "checks_pass" and not TEST_FAIL.search(out) and not ev.is_error):
|
|
79
|
+
if rules.NARROWED_TEST.search(seg) and category == "tests_pass":
|
|
80
|
+
claimed = re.search(r"(\d+)\s*(个|项)?\s*(测试|用例|tests?)", claim_text or "")
|
|
81
|
+
passed = re.search(r"\b(\d+) passed\b", summ)
|
|
82
|
+
if claimed and passed and claimed.group(1) == passed.group(1):
|
|
83
|
+
return "backed", f"narrowed run matches the claimed count ({passed.group(1)} passed): {seg[:60]}"
|
|
84
|
+
return "unverifiable", "narrowed test run reported as passing"
|
|
85
|
+
return "backed", f"test output confirms: {seg[:80]}"
|
|
86
|
+
if rules.PIPE_SWALLOW.search(seg) and "pipefail" not in seg:
|
|
87
|
+
return "unverifiable", "test piped through tail/head and output shows no pass/fail line"
|
|
88
|
+
if ev.is_error:
|
|
89
|
+
return "unbacked", "test command errored"
|
|
90
|
+
return "unverifiable", "test ran but output shows no pass/fail line"
|
|
91
|
+
if category == "pushed":
|
|
92
|
+
if PUSH_FAIL.search(out):
|
|
93
|
+
return "contradicted", "push output shows an error"
|
|
94
|
+
return ("unbacked", "push command errored") if ev.is_error else ("backed", f"push confirmed: {seg[:80]}")
|
|
95
|
+
if ev.is_error:
|
|
96
|
+
return "unbacked", f"verification command errored: {seg[:60]}"
|
|
97
|
+
if category in ("build_ok", "rendered") and re.search(r"(Traceback|\berror\b|fatal)", out, re.I) and not re.search(r"0 errors?|no errors?", out, re.I):
|
|
98
|
+
return "contradicted", "verification output contains errors"
|
|
99
|
+
if rules.PIPE_SWALLOW.search(seg) and "pipefail" not in seg and not out.strip():
|
|
100
|
+
return "unverifiable", "piped verification with empty output"
|
|
101
|
+
return "backed", f"verified by: {seg[:80]}"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def audit_turn(turn: Turn):
|
|
105
|
+
out = []
|
|
106
|
+
ev = turn.events
|
|
107
|
+
for i, e in enumerate(ev):
|
|
108
|
+
if e.kind != "text":
|
|
109
|
+
continue
|
|
110
|
+
for hit in rules.find_claims(e.text):
|
|
111
|
+
before = ev[:i]
|
|
112
|
+
edit_idx = max((j for j, x in enumerate(before) if _is_edit(x)), default=-1)
|
|
113
|
+
after = [x for x in before[edit_idx + 1:] if x.kind == "tool"] if edit_idx >= 0 else []
|
|
114
|
+
tools_after = [f"{x.name}: {x.text[:240]}" + (" [ERR]" if x.is_error else "") for x in after]
|
|
115
|
+
last_edit = f"{ev[edit_idx].name}: {ev[edit_idx].text[:400]}" if edit_idx >= 0 else ""
|
|
116
|
+
ctx_start = max(0, e.text.find(hit.sentence) - 150)
|
|
117
|
+
context = e.text[ctx_start: ctx_start + 150 + len(hit.sentence) + 150].replace("\n", " ")
|
|
118
|
+
if not hit.evidence_kinds:
|
|
119
|
+
verdict, reason = "out_of_scope", "fuzzy category not audited in v1"
|
|
120
|
+
elif edit_idx < 0:
|
|
121
|
+
verdict, reason = "not_this_turn", "no code edit in this turn before the claim"
|
|
122
|
+
elif any(x.name in rules.OPAQUE_TOOLS for x in after):
|
|
123
|
+
verdict, reason = "unverifiable", "a subagent/background/terminal tool ran after the last edit"
|
|
124
|
+
elif rules.REMOTE_RUN.search(hit.sentence):
|
|
125
|
+
verdict, reason = "unverifiable", "claim is about a remote or background run; local edits do not invalidate it and its evidence is off-transcript"
|
|
126
|
+
else:
|
|
127
|
+
verifying = [(x, rules.verifying_segments(x.text, hit.evidence_kinds)) for x in after if x.name == "Bash"]
|
|
128
|
+
verifying = [(x, segs) for x, segs in verifying if segs]
|
|
129
|
+
edit_ev = ev[edit_idx]
|
|
130
|
+
if not verifying and edit_ev.name == "Bash":
|
|
131
|
+
segs = rules.verifying_segments(edit_ev.text, hit.evidence_kinds, after_edit_only=True)
|
|
132
|
+
if segs:
|
|
133
|
+
verifying = [(edit_ev, segs)]
|
|
134
|
+
if not verifying and hit.category in ("fixed", "rendered", "checks_pass") and any(
|
|
135
|
+
(x.name == "Read" and IMAGE_READ.search(x.text)) or (x.name == "Bash" and VISUAL_CMD.search(x.text)) for x in after):
|
|
136
|
+
verdict, reason = "weak", "judged by looking at a rendered image or snapshot after the edit; not checkable mechanically"
|
|
137
|
+
elif not verifying:
|
|
138
|
+
verdict, reason = "unbacked", "no verification command after the last edit"
|
|
139
|
+
else:
|
|
140
|
+
last, segs = verifying[-1]
|
|
141
|
+
seg = segs[-1].text
|
|
142
|
+
verdict, reason = judge_output(hit.category, seg, last, hit.sentence)
|
|
143
|
+
vb = locals().get("seg", "") if verdict in ("backed", "unverifiable", "contradicted") and edit_idx >= 0 else ""
|
|
144
|
+
out_snip = (locals().get("last").result[:300].replace("\n", " ") if vb and locals().get("last") else "")
|
|
145
|
+
out.append(Finding(turn.session, turn.index, turn.version, turn.agent, turn.model, e.ts, hit.category, verdict, reason,
|
|
146
|
+
hit.sentence[:300], last_edit, tools_after[-8:], context[:600], vb[:200], out_snip, getattr(turn, "project", "")))
|
|
147
|
+
seg = ""; last = None
|
|
148
|
+
return out
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def audit_turns(turns):
|
|
152
|
+
findings = []
|
|
153
|
+
for t in turns:
|
|
154
|
+
findings.extend(audit_turn(t))
|
|
155
|
+
return findings
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def summarize(findings):
|
|
159
|
+
from collections import Counter
|
|
160
|
+
c = Counter(f.verdict for f in findings)
|
|
161
|
+
cat = Counter((f.category, f.verdict) for f in findings)
|
|
162
|
+
return {"total_claims": len(findings), "by_verdict": dict(c), "by_category": {f"{k[0]}/{k[1]}": v for k, v in cat.items()}}
|
honestly/claude_code.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Parse Claude Code session logs (~/.claude/projects/**/*.jsonl) into turns.
|
|
2
|
+
|
|
3
|
+
A turn = everything the assistant did between two human prompts.
|
|
4
|
+
Events are kept in order: text blocks, tool calls (with result status),
|
|
5
|
+
so rules can ask "what happened after the last edit?".
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
import glob
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Event:
|
|
16
|
+
kind: str # text | tool
|
|
17
|
+
name: str = "" # tool name for kind=tool
|
|
18
|
+
text: str = "" # text block, or command/path summary for tools
|
|
19
|
+
tool_use_id: str = ""
|
|
20
|
+
is_error: bool | None = None
|
|
21
|
+
ts: str = ""
|
|
22
|
+
result: str = ""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Turn:
|
|
27
|
+
session: str
|
|
28
|
+
index: int
|
|
29
|
+
version: str
|
|
30
|
+
prompt: str
|
|
31
|
+
events: list = field(default_factory=list)
|
|
32
|
+
ts: str = ""
|
|
33
|
+
agent: str = "claude-code"
|
|
34
|
+
model: str = ""
|
|
35
|
+
project: str = ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _summarize_input(name: str, inp) -> str:
|
|
39
|
+
if not isinstance(inp, dict):
|
|
40
|
+
return ""
|
|
41
|
+
if name == "Bash":
|
|
42
|
+
return inp.get("command", "") or ""
|
|
43
|
+
for k in ("file_path", "path", "notebook_path", "url", "pattern", "query", "prompt", "description"):
|
|
44
|
+
if k in inp:
|
|
45
|
+
return str(inp[k])
|
|
46
|
+
return ""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def iter_sessions(root: str | None = None):
|
|
50
|
+
root = root or os.path.expanduser("~/.claude/projects")
|
|
51
|
+
for path in sorted(glob.glob(os.path.join(root, "*", "*.jsonl"))):
|
|
52
|
+
yield path
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_session(path: str):
|
|
56
|
+
session = os.path.splitext(os.path.basename(path))[0]
|
|
57
|
+
turns: list[Turn] = []
|
|
58
|
+
cur: Turn | None = None
|
|
59
|
+
results: dict[str, tuple] = {} # tool_use_id -> (is_error, text)
|
|
60
|
+
pending: list[Event] = []
|
|
61
|
+
version = ""
|
|
62
|
+
project = ""
|
|
63
|
+
with open(path, encoding="utf-8", errors="ignore") as fh:
|
|
64
|
+
for line in fh:
|
|
65
|
+
try:
|
|
66
|
+
r = json.loads(line)
|
|
67
|
+
except Exception:
|
|
68
|
+
continue
|
|
69
|
+
if not isinstance(r, dict):
|
|
70
|
+
continue
|
|
71
|
+
if r.get("isSidechain"):
|
|
72
|
+
continue # subagent lines live elsewhere; keep main thread only
|
|
73
|
+
t = r.get("type")
|
|
74
|
+
version = r.get("version") if isinstance(r.get("version"), str) else version
|
|
75
|
+
project = r.get("cwd") if isinstance(r.get("cwd"), str) else project
|
|
76
|
+
msg = r.get("message") if isinstance(r.get("message"), dict) else {}
|
|
77
|
+
origin = r.get("origin") if isinstance(r.get("origin"), dict) else {}
|
|
78
|
+
ts = str(r.get("timestamp") or "")
|
|
79
|
+
content = msg.get("content")
|
|
80
|
+
if t == "user":
|
|
81
|
+
human = origin.get("kind") == "human" or isinstance(content, str)
|
|
82
|
+
if isinstance(content, list):
|
|
83
|
+
for b in content:
|
|
84
|
+
if isinstance(b, dict) and b.get("type") == "tool_result":
|
|
85
|
+
rc = b.get("content")
|
|
86
|
+
if isinstance(rc, list):
|
|
87
|
+
rc = " ".join(str(x.get("text") or "") for x in rc if isinstance(x, dict))
|
|
88
|
+
results[b.get("tool_use_id", "")] = (bool(b.get("is_error")), (rc or "")[:3000])
|
|
89
|
+
if any(isinstance(b, dict) and b.get("type") == "text" for b in content) and origin.get("kind") == "human":
|
|
90
|
+
human = True
|
|
91
|
+
if human:
|
|
92
|
+
prompt = content if isinstance(content, str) else " ".join(
|
|
93
|
+
str(b.get("text") or "") for b in content if isinstance(b, dict) and b.get("type") == "text")
|
|
94
|
+
if cur is not None:
|
|
95
|
+
turns.append(cur)
|
|
96
|
+
cur = Turn(session=session, index=len(turns), version=version, prompt=prompt[:200], ts=ts, project=project)
|
|
97
|
+
elif t == "assistant" and isinstance(content, list):
|
|
98
|
+
if cur is None:
|
|
99
|
+
cur = Turn(session=session, index=0, version=version, prompt="", ts=ts)
|
|
100
|
+
if isinstance(msg.get("model"), str) and not cur.model:
|
|
101
|
+
cur.model = msg["model"]
|
|
102
|
+
for b in content:
|
|
103
|
+
if not isinstance(b, dict):
|
|
104
|
+
continue
|
|
105
|
+
if b.get("type") == "text":
|
|
106
|
+
cur.events.append(Event("text", text=str(b.get("text") or ""), ts=ts))
|
|
107
|
+
elif b.get("type") == "tool_use":
|
|
108
|
+
cur.events.append(Event("tool", name=str(b.get("name") or ""), text=_summarize_input(str(b.get("name") or ""), b.get("input")),
|
|
109
|
+
tool_use_id=str(b.get("id") or ""), ts=ts))
|
|
110
|
+
if cur is not None:
|
|
111
|
+
turns.append(cur)
|
|
112
|
+
for turn in turns:
|
|
113
|
+
for e in turn.events:
|
|
114
|
+
if e.kind == "tool" and e.tool_use_id in results:
|
|
115
|
+
e.is_error, e.result = results[e.tool_use_id]
|
|
116
|
+
return turns
|
honestly/cli.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import random
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
from dataclasses import asdict
|
|
9
|
+
from . import claude_code, codex
|
|
10
|
+
from .audit import audit_turns
|
|
11
|
+
from .report import text_report, write_card, colorize, want_color
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _since_cutoff(spec):
|
|
15
|
+
if not spec:
|
|
16
|
+
return None
|
|
17
|
+
import datetime as dt
|
|
18
|
+
m = re.fullmatch(r"(\d+)([dwm])", spec.strip())
|
|
19
|
+
if not m:
|
|
20
|
+
raise SystemExit("--since takes a number followed by d, w or m, e.g. 30d")
|
|
21
|
+
days = int(m.group(1)) * {"d": 1, "w": 7, "m": 30}[m.group(2)]
|
|
22
|
+
return (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%S")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def collect(root_claude=None, root_codex=None, agents=("claude-code", "codex"), since=None):
|
|
26
|
+
cutoff = _since_cutoff(since)
|
|
27
|
+
findings, sessions = [], 0
|
|
28
|
+
if "claude-code" in agents:
|
|
29
|
+
for path in claude_code.iter_sessions(root_claude):
|
|
30
|
+
sessions += 1
|
|
31
|
+
findings.extend(audit_turns(claude_code.parse_session(path)))
|
|
32
|
+
if "codex" in agents:
|
|
33
|
+
for path in codex.iter_sessions(root_codex):
|
|
34
|
+
sessions += 1
|
|
35
|
+
findings.extend(audit_turns(codex.parse_session(path)))
|
|
36
|
+
if cutoff:
|
|
37
|
+
findings = [f for f in findings if (f.ts or "9999") >= cutoff]
|
|
38
|
+
seen, deduped = set(), []
|
|
39
|
+
for f in findings:
|
|
40
|
+
key = (f.category, f.claim, f.last_edit[:60], str(f.ts or "")[:16])
|
|
41
|
+
if key in seen:
|
|
42
|
+
continue
|
|
43
|
+
seen.add(key); deduped.append(f)
|
|
44
|
+
return deduped, sessions, len(findings) - len(deduped)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cmd_audit(a):
|
|
48
|
+
agents = tuple(a.agent) if a.agent else ("claude-code", "codex")
|
|
49
|
+
findings, sessions, dups = collect(a.root, a.codex_root, agents, a.since)
|
|
50
|
+
if a.json:
|
|
51
|
+
with open(a.json, "w", encoding="utf-8") as fh:
|
|
52
|
+
for f in findings:
|
|
53
|
+
d = asdict(f); d["label"] = f.label
|
|
54
|
+
fh.write(json.dumps(d, ensure_ascii=False) + "\n")
|
|
55
|
+
print(f"wrote {len(findings)} findings to {a.json}", file=sys.stderr)
|
|
56
|
+
if a.summary_json:
|
|
57
|
+
from .audit import summarize
|
|
58
|
+
s = summarize(findings); s["sessions"] = sessions; s["duplicates_removed_forks"] = dups
|
|
59
|
+
print(json.dumps(s, ensure_ascii=False, indent=2))
|
|
60
|
+
return
|
|
61
|
+
report = text_report(findings, sessions, dups)
|
|
62
|
+
print(colorize(report) if want_color() else report)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def cmd_share(a):
|
|
66
|
+
findings, sessions, _ = collect(a.root, a.codex_root, since=a.since)
|
|
67
|
+
out = a.out or os.path.expanduser("~/.honestly")
|
|
68
|
+
written = write_card(findings, sessions, out)
|
|
69
|
+
for p in written:
|
|
70
|
+
print(p)
|
|
71
|
+
if not any(p.endswith(".png") for p in written):
|
|
72
|
+
print("PNG not written: no rsvg-convert, Chrome or Edge found. The .svg above opens in any browser; "
|
|
73
|
+
"screenshot it there, or install one of those and run honestly share again.", file=sys.stderr)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def cmd_explain(a):
|
|
77
|
+
findings, _, _ = collect(a.root, a.codex_root, since=a.since)
|
|
78
|
+
flagged = [f for f in findings if f.verdict in ("unbacked", "contradicted")]
|
|
79
|
+
if not flagged:
|
|
80
|
+
print("nothing flagged"); return
|
|
81
|
+
if a.n < 1 or a.n > len(flagged):
|
|
82
|
+
print(f"pick 1..{len(flagged)}"); return
|
|
83
|
+
f = flagged[a.n - 1]
|
|
84
|
+
print(f"[{a.n}/{len(flagged)}] {f.label} · {f.category} · {f.agent} {f.model or f.version} · session {f.session[:8]} turn {f.turn} · {f.ts[:19]}")
|
|
85
|
+
print(f"project: {f.project or '-'}")
|
|
86
|
+
print(f"claim: {f.claim}")
|
|
87
|
+
print(f"context: …{f.context}…")
|
|
88
|
+
print(f"last code edit: {f.last_edit or '-'}")
|
|
89
|
+
print("tools after it:" if f.tools_after_edit else "tools after it: (none)")
|
|
90
|
+
for t in f.tools_after_edit:
|
|
91
|
+
print(f" - {t}")
|
|
92
|
+
print(f"why: {f.reason}")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def cmd_catches(a):
|
|
96
|
+
from .hooks import catches_report
|
|
97
|
+
print(catches_report(a.days))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def cmd_guard(a):
|
|
101
|
+
from .hooks import guard
|
|
102
|
+
print(guard(a.action, dry_run=a.dry_run))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def cmd_hook(a):
|
|
106
|
+
from .hooks import run_hook
|
|
107
|
+
sys.exit(run_hook(a.which))
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def cmd_sample(a):
|
|
111
|
+
rows = [json.loads(l) for l in open(a.findings, encoding="utf-8")]
|
|
112
|
+
rows = [r for r in rows if r["verdict"] == a.verdict and not r["session"].startswith(tuple(a.exclude))]
|
|
113
|
+
random.Random(a.seed).shuffle(rows)
|
|
114
|
+
per, picked = {}, []
|
|
115
|
+
for r in rows:
|
|
116
|
+
if per.get(r["session"], 0) >= a.per_session:
|
|
117
|
+
continue
|
|
118
|
+
per[r["session"]] = per.get(r["session"], 0) + 1
|
|
119
|
+
picked.append(r)
|
|
120
|
+
if len(picked) >= a.n:
|
|
121
|
+
break
|
|
122
|
+
with open(a.out, "w", encoding="utf-8") as fh:
|
|
123
|
+
fh.write(f"# Label sheet: {len(picked)} claims judged `{a.verdict}` by the audit\n\n")
|
|
124
|
+
fh.write("For each item answer: **TP** (the audit is right), **FP** (the audit is wrong), or **UNSURE**.\n\n")
|
|
125
|
+
for i, r in enumerate(picked, 1):
|
|
126
|
+
fh.write(f"## {i}. [{r['category']}] session {r['session'][:8]} turn {r['turn']} ({r.get('agent','')} {r['version']})\n")
|
|
127
|
+
fh.write(f"- claim: {r['claim']}\n- context: …{r['context']}…\n- last edit: {r['last_edit'] or '(none)'}\n")
|
|
128
|
+
fh.write("- tools after last edit: " + ("; ".join(r['tools_after_edit']) if r['tools_after_edit'] else "(none)") + "\n")
|
|
129
|
+
if r.get("verified_by"):
|
|
130
|
+
fh.write(f"- verifying command segment: {r['verified_by']}\n- its output (first 300 chars): {r.get('output','')}\n")
|
|
131
|
+
fh.write(f"- audit reason: {r['reason']}\n- label: \n\n")
|
|
132
|
+
with open(a.out.replace('.md', '.jsonl'), "w", encoding="utf-8") as fh:
|
|
133
|
+
for r in picked:
|
|
134
|
+
fh.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
135
|
+
print(f"wrote {len(picked)} items to {a.out}")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def main(argv=None):
|
|
139
|
+
for stream in (sys.stdout, sys.stderr): # Windows redirects default to the locale code page
|
|
140
|
+
if hasattr(stream, "reconfigure"):
|
|
141
|
+
try:
|
|
142
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
143
|
+
except Exception:
|
|
144
|
+
pass
|
|
145
|
+
p = argparse.ArgumentParser(prog="honestly", description="Audit how often your coding agent verified what it claimed.")
|
|
146
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
147
|
+
pa = sub.add_parser("audit", help="scan Claude Code / Codex session logs and print the report")
|
|
148
|
+
pa.add_argument("--root", default=None, help="Claude Code projects dir (default ~/.claude/projects)")
|
|
149
|
+
pa.add_argument("--codex-root", default=None, help="Codex sessions dir (default ~/.codex/sessions)")
|
|
150
|
+
pa.add_argument("--agent", action="append", choices=["claude-code", "codex"], help="limit to one agent")
|
|
151
|
+
pa.add_argument("--since", default=None, help="only claims from the last N days/weeks/months, e.g. 30d, 8w, 3m")
|
|
152
|
+
pa.add_argument("--json", default=None, help="also write every finding as JSONL")
|
|
153
|
+
pa.add_argument("--summary-json", action="store_true", help="print machine-readable counts instead of the report")
|
|
154
|
+
pa.set_defaults(fn=cmd_audit)
|
|
155
|
+
ps = sub.add_parser("share", help="write a shareable card (SVG, and PNG when a converter is available)")
|
|
156
|
+
ps.add_argument("--root", default=None, help="Claude Code projects dir (default ~/.claude/projects)")
|
|
157
|
+
ps.add_argument("--codex-root", default=None, help="Codex sessions dir (default ~/.codex/sessions)")
|
|
158
|
+
ps.add_argument("--out", default=None, help="output directory (default ~/.honestly)")
|
|
159
|
+
ps.add_argument("--since", default=None, help="e.g. 30d, 8w, 3m")
|
|
160
|
+
ps.set_defaults(fn=cmd_share)
|
|
161
|
+
pe = sub.add_parser("explain", help="show the full timeline behind the n-th flagged claim")
|
|
162
|
+
pe.add_argument("n", type=int); pe.add_argument("--root", default=None); pe.add_argument("--codex-root", default=None); pe.add_argument("--since", default=None)
|
|
163
|
+
pe.set_defaults(fn=cmd_explain)
|
|
164
|
+
pc = sub.add_parser("catches", help="what guard stopped recently (reads ~/.honestly/catches.jsonl)")
|
|
165
|
+
pc.add_argument("--days", type=int, default=7, help="window in days (default 7)")
|
|
166
|
+
pc.set_defaults(fn=cmd_catches)
|
|
167
|
+
pg = sub.add_parser("guard", help="install / uninstall the Claude Code hooks")
|
|
168
|
+
pg.add_argument("action", choices=["install", "uninstall", "status"])
|
|
169
|
+
pg.add_argument("--dry-run", action="store_true", help="print the resulting hooks section without writing")
|
|
170
|
+
pg.set_defaults(fn=cmd_guard)
|
|
171
|
+
ph = sub.add_parser("hook", help="(internal) run as a Claude Code hook; reads JSON on stdin")
|
|
172
|
+
ph.add_argument("which", choices=["stop", "pre"])
|
|
173
|
+
ph.set_defaults(fn=cmd_hook)
|
|
174
|
+
pm = sub.add_parser("sample", help="(maintainers) draw a labeling sample from a findings JSONL")
|
|
175
|
+
pm.add_argument("findings"); pm.add_argument("--verdict", default="unbacked"); pm.add_argument("--n", type=int, default=100)
|
|
176
|
+
pm.add_argument("--per-session", type=int, default=8); pm.add_argument("--seed", type=int, default=42)
|
|
177
|
+
pm.add_argument("--exclude", nargs="*", default=[]); pm.add_argument("--out", default="gate/label_sheet.md")
|
|
178
|
+
pm.set_defaults(fn=cmd_sample)
|
|
179
|
+
a = p.parse_args(argv)
|
|
180
|
+
a.fn(a)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__":
|
|
184
|
+
main()
|
honestly/codex.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Parse Codex (OpenAI) session logs (~/.codex/sessions/**/rollout-*.jsonl) into turns.
|
|
2
|
+
|
|
3
|
+
Codex writes `response_item` records: messages (user/assistant/developer), `custom_tool_call`
|
|
4
|
+
(name "exec" with a JS-ish input holding cmd:"...", or "apply_patch" with a patch body) and their
|
|
5
|
+
`*_output` records, plus `function_call`/`function_call_output` for shell-style tools.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
import glob
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
from .claude_code import Event, Turn
|
|
13
|
+
|
|
14
|
+
CMD_IN_INPUT = re.compile(r"cmd\s*:\s*\"((?:\\.|[^\"\\])*)\"")
|
|
15
|
+
PATCH_FILES = re.compile(r"^\*\*\* (?:Update|Add|Delete) File: (.+)$", re.M)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def iter_sessions(root: str | None = None):
|
|
19
|
+
root = root or os.path.expanduser("~/.codex/sessions")
|
|
20
|
+
yield from sorted(glob.glob(os.path.join(root, "**", "*.jsonl"), recursive=True))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _unescape(s: str) -> str:
|
|
24
|
+
try:
|
|
25
|
+
return json.loads('"' + s + '"')
|
|
26
|
+
except Exception:
|
|
27
|
+
return s.replace('\\"', '"').replace("\\n", "\n")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _dig(d, *keys):
|
|
31
|
+
"""d[k1][k2]... when every level is a dict and the leaf is a string, else None."""
|
|
32
|
+
for k in keys:
|
|
33
|
+
d = d.get(k) if isinstance(d, dict) else None
|
|
34
|
+
return d if isinstance(d, str) else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _command_from_call(name: str, payload: dict) -> tuple[str, str]:
|
|
38
|
+
"""Return (tool_kind, text) where tool_kind is Bash / Edit / other."""
|
|
39
|
+
inp0 = payload.get("input")
|
|
40
|
+
if isinstance(inp0, str) and CMD_IN_INPUT.search(inp0):
|
|
41
|
+
name = "exec" # the desktop app wraps shell calls in a JS runner; the cmd is what matters
|
|
42
|
+
if name in ("exec", "shell", "shell_command", "exec_command", "container.exec", "local_shell"): # desktop app + Codex CLI names
|
|
43
|
+
inp = payload.get("input")
|
|
44
|
+
args = payload.get("arguments")
|
|
45
|
+
if isinstance(inp, str):
|
|
46
|
+
m = CMD_IN_INPUT.search(inp)
|
|
47
|
+
if m:
|
|
48
|
+
return "Bash", _unescape(m.group(1))
|
|
49
|
+
return "Bash", inp
|
|
50
|
+
if isinstance(args, str):
|
|
51
|
+
try:
|
|
52
|
+
a = json.loads(args)
|
|
53
|
+
except Exception:
|
|
54
|
+
return "Bash", args
|
|
55
|
+
if not isinstance(a, dict):
|
|
56
|
+
return "Bash", args
|
|
57
|
+
cmd = a.get("cmd") or a.get("command") or ""
|
|
58
|
+
if isinstance(cmd, list):
|
|
59
|
+
cmd = " ".join(str(x) for x in cmd if x is not None)
|
|
60
|
+
return "Bash", str(cmd)
|
|
61
|
+
return "Bash", ""
|
|
62
|
+
if name == "apply_patch":
|
|
63
|
+
body = payload.get("input") or payload.get("arguments") or ""
|
|
64
|
+
if isinstance(body, str) and body.startswith("{"):
|
|
65
|
+
try:
|
|
66
|
+
body = json.loads(body).get("patch", body)
|
|
67
|
+
except Exception:
|
|
68
|
+
pass
|
|
69
|
+
files = PATCH_FILES.findall(body if isinstance(body, str) else "")
|
|
70
|
+
return "Edit", " ".join(files) or "patch"
|
|
71
|
+
return name or "tool", ""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def parse_session(path: str):
|
|
75
|
+
session = os.path.splitext(os.path.basename(path))[0].replace("rollout-", "")[:40]
|
|
76
|
+
turns: list[Turn] = []
|
|
77
|
+
cur: Turn | None = None
|
|
78
|
+
model = ""
|
|
79
|
+
project = ""
|
|
80
|
+
pending: dict[str, Event] = {} # call_id -> event, to attach outputs
|
|
81
|
+
with open(path, encoding="utf-8", errors="ignore") as fh:
|
|
82
|
+
for line in fh:
|
|
83
|
+
try:
|
|
84
|
+
r = json.loads(line)
|
|
85
|
+
except Exception:
|
|
86
|
+
continue
|
|
87
|
+
if not isinstance(r, dict):
|
|
88
|
+
continue
|
|
89
|
+
t = r.get("type")
|
|
90
|
+
p = r.get("payload") if isinstance(r.get("payload"), dict) else {}
|
|
91
|
+
ts = str(r.get("timestamp") or "")
|
|
92
|
+
if t in ("turn_context", "session_meta") and isinstance(p.get("cwd"), str):
|
|
93
|
+
project = p["cwd"]
|
|
94
|
+
if t in ("turn_context", "world_state", "session_meta"):
|
|
95
|
+
m = (_dig(p, "model") or _dig(p, "state", "collaboration_mode", "model")
|
|
96
|
+
or _dig(p, "collaboration_mode", "model") or _dig(p, "model_name"))
|
|
97
|
+
model = m or model
|
|
98
|
+
if cur is not None and not cur.model:
|
|
99
|
+
cur.model = model; cur.version = model
|
|
100
|
+
continue
|
|
101
|
+
item = p.get("item") if isinstance(p.get("item"), dict) else {}
|
|
102
|
+
if t == "event_msg" and p.get("type") == "item_completed" and item.get("type") == "UserMessage":
|
|
103
|
+
text = " ".join(str(c.get("text") or "") for c in (item.get("content") or []) if isinstance(c, dict))
|
|
104
|
+
if cur is not None:
|
|
105
|
+
turns.append(cur)
|
|
106
|
+
cur = Turn(session=session, index=len(turns), version=model, prompt=text[:200], ts=ts, agent="codex", model=model, project=project)
|
|
107
|
+
continue
|
|
108
|
+
if t != "response_item":
|
|
109
|
+
continue
|
|
110
|
+
kind = p.get("type")
|
|
111
|
+
if cur is None:
|
|
112
|
+
cur = Turn(session=session, index=0, version=model, prompt="", ts=ts, agent="codex", model=model)
|
|
113
|
+
if kind == "message" and p.get("role") == "assistant":
|
|
114
|
+
text = " ".join(str(c.get("text") or "") for c in (p.get("content") or []) if isinstance(c, dict))
|
|
115
|
+
if text.strip():
|
|
116
|
+
cur.events.append(Event("text", text=text, ts=ts))
|
|
117
|
+
elif kind in ("custom_tool_call", "function_call"):
|
|
118
|
+
tool, text = _command_from_call(str(p.get("name") or ""), p)
|
|
119
|
+
ev = Event("tool", name=tool, text=text, tool_use_id=str(p.get("call_id") or ""), ts=ts)
|
|
120
|
+
cur.events.append(ev)
|
|
121
|
+
if ev.tool_use_id:
|
|
122
|
+
pending[ev.tool_use_id] = ev
|
|
123
|
+
elif kind in ("custom_tool_call_output", "function_call_output"):
|
|
124
|
+
out = p.get("output")
|
|
125
|
+
if isinstance(out, list):
|
|
126
|
+
out = " ".join(str(c.get("text") or "") for c in out if isinstance(c, dict))
|
|
127
|
+
out = str(out or "")
|
|
128
|
+
ev = pending.get(str(p.get("call_id") or ""))
|
|
129
|
+
if ev is not None:
|
|
130
|
+
ev.result = out[:3000]
|
|
131
|
+
ev.is_error = bool(re.search(r"(exit(?:ed with)? code:? [1-9]|\"exit_code\":\s*[1-9]|Traceback|command not found|No such file)", out[:500], re.I))
|
|
132
|
+
if cur is not None:
|
|
133
|
+
turns.append(cur)
|
|
134
|
+
return turns
|