repeat-o-scope 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repeat_o_scope/__init__.py +0 -0
- repeat_o_scope/audit.py +136 -0
- repeat_o_scope/cli.py +248 -0
- repeat_o_scope/cluster.py +76 -0
- repeat_o_scope/declined.py +70 -0
- repeat_o_scope/doctor.py +211 -0
- repeat_o_scope/findings.py +71 -0
- repeat_o_scope/install.py +103 -0
- repeat_o_scope/judge.py +67 -0
- repeat_o_scope/judges.py +218 -0
- repeat_o_scope/redaction.py +16 -0
- repeat_o_scope/render.py +259 -0
- repeat_o_scope/skill/SKILL.md +84 -0
- repeat_o_scope/transcripts.py +190 -0
- repeat_o_scope/waste.py +262 -0
- repeat_o_scope/worklog.py +233 -0
- repeat_o_scope-0.1.0.dist-info/METADATA +255 -0
- repeat_o_scope-0.1.0.dist-info/RECORD +21 -0
- repeat_o_scope-0.1.0.dist-info/WHEEL +4 -0
- repeat_o_scope-0.1.0.dist-info/entry_points.txt +4 -0
- repeat_o_scope-0.1.0.dist-info/licenses/LICENSE +21 -0
repeat_o_scope/render.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Turns a report into markdown for people or JSON for the skill."""
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
|
|
8
|
+
from repeat_o_scope.audit import Audit
|
|
9
|
+
from repeat_o_scope.doctor import Check
|
|
10
|
+
from repeat_o_scope.findings import Finding, Report
|
|
11
|
+
from repeat_o_scope.waste import Measurement, Tally
|
|
12
|
+
|
|
13
|
+
_CHECK_LABELS = {
|
|
14
|
+
'effort': 'Effort',
|
|
15
|
+
'skill_overrides': 'Skill overrides',
|
|
16
|
+
'skills': 'Enabled skills',
|
|
17
|
+
'status_line': 'Status line',
|
|
18
|
+
'scope_rule': 'Scope rule',
|
|
19
|
+
'read_once_rule': 'Read-once rule',
|
|
20
|
+
'idle_compact': 'Idle compact',
|
|
21
|
+
'autocompact_override': 'Auto-compact override',
|
|
22
|
+
}
|
|
23
|
+
_KIND_LABELS = {
|
|
24
|
+
'refused_edit': 'Refused edits',
|
|
25
|
+
'reread': 'Re-reads',
|
|
26
|
+
'failed': 'Failed calls',
|
|
27
|
+
'denied': 'Denied tools',
|
|
28
|
+
'bare_yes': 'Bare go-aheads',
|
|
29
|
+
'cache_rebuild': 'Expired cache',
|
|
30
|
+
'long_context': 'Past half context',
|
|
31
|
+
}
|
|
32
|
+
_SHOWN_VARIANTS = 3
|
|
33
|
+
_MAX_SHOWN_CHARS = 120
|
|
34
|
+
_TITLES = {
|
|
35
|
+
'prompts': ('Prompts you keep typing', 'prompts'),
|
|
36
|
+
'commands': ('Commands you keep running', 'commands'),
|
|
37
|
+
'fails': ('Tool calls that keep failing', 'failures'),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def json_document(report: Report, *, top: int) -> str:
|
|
42
|
+
"""Returns the report as JSON with every finding; `top` only limits markdown."""
|
|
43
|
+
doc = {
|
|
44
|
+
'report': report.name,
|
|
45
|
+
'days': report.days,
|
|
46
|
+
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
47
|
+
'jev': report.jev,
|
|
48
|
+
'totals': report.totals,
|
|
49
|
+
'findings': [finding_dict(f) for f in report.findings],
|
|
50
|
+
}
|
|
51
|
+
return json.dumps(doc, ensure_ascii=False, indent=2)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def finding_dict(f: Finding) -> dict:
|
|
55
|
+
"""Returns the finding as a dict; `cost` and `share` only when it has them, as only audit findings do."""
|
|
56
|
+
return {k: v for k, v in dataclasses.asdict(f).items() if v is not None or k not in ('cost', 'share')}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def markdown(report: Report, *, top: int) -> str:
|
|
60
|
+
"""Returns the report as markdown, showing at most `top` findings per section."""
|
|
61
|
+
if report.name == 'asks':
|
|
62
|
+
return _asks(report, top=top)
|
|
63
|
+
title, unit = _TITLES[report.name]
|
|
64
|
+
t = report.totals
|
|
65
|
+
text = _section(
|
|
66
|
+
f'# {title}',
|
|
67
|
+
[f for f in report.findings if f.kind != 'denial'],
|
|
68
|
+
summary=f'{t["items"]} {unit}, {t["groups"]} groups, {t["one_offs"]} one-offs.',
|
|
69
|
+
days=report.days,
|
|
70
|
+
note=report.note,
|
|
71
|
+
top=top,
|
|
72
|
+
)
|
|
73
|
+
if report.name == 'fails':
|
|
74
|
+
text += '\n\n' + _section(
|
|
75
|
+
'## Denied',
|
|
76
|
+
[f for f in report.findings if f.kind == 'denial'],
|
|
77
|
+
summary=f'{t["denials"]} denials, {t["denial_groups"]} groups, {t["denial_one_offs"]} one-offs.',
|
|
78
|
+
days=report.days,
|
|
79
|
+
note='',
|
|
80
|
+
top=top,
|
|
81
|
+
)
|
|
82
|
+
return text
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def measurement_markdown(m: Measurement, *, price: float | None) -> str:
|
|
86
|
+
"""Returns the measurement as a markdown table, with euros only when `price` is set."""
|
|
87
|
+
header = _measurement_header(price=price)
|
|
88
|
+
lines = [
|
|
89
|
+
'# Wasted turns',
|
|
90
|
+
'',
|
|
91
|
+
f'_{_measurement_headline(m)}_',
|
|
92
|
+
'',
|
|
93
|
+
'| ' + ' | '.join(header) + ' |',
|
|
94
|
+
'|---|' + '---:|' * (len(header) - 1),
|
|
95
|
+
]
|
|
96
|
+
lines += ['| ' + ' | '.join(row) + ' |' for row in _measurement_rows(m, price=price)]
|
|
97
|
+
return '\n'.join(lines)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def measurement_table(m: Measurement, *, price: float | None) -> str:
|
|
101
|
+
"""Returns the measurement as a rich table for a terminal, with euros only when `price` is set."""
|
|
102
|
+
from rich.console import Console
|
|
103
|
+
from rich.table import Table
|
|
104
|
+
|
|
105
|
+
table = Table(title='Wasted turns', caption=_measurement_headline(m))
|
|
106
|
+
for i, column in enumerate(_measurement_header(price=price)):
|
|
107
|
+
table.add_column(column, justify='left' if i == 0 else 'right')
|
|
108
|
+
for row in _measurement_rows(m, price=price):
|
|
109
|
+
table.add_row(*row)
|
|
110
|
+
console = Console()
|
|
111
|
+
with console.capture() as captured:
|
|
112
|
+
console.print(table)
|
|
113
|
+
return captured.get()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _measurement_headline(m: Measurement) -> str:
|
|
117
|
+
# Long-context and rebuild turns waste only part of a call, so they are not counted as wasted turns.
|
|
118
|
+
long, rebuilt = m.waste['long_context'].turns, m.waste['cache_rebuild'].turns
|
|
119
|
+
wasted = sum(t.turns for t in m.waste.values()) - long - rebuilt
|
|
120
|
+
share = _share(sum(t.weight for t in m.waste.values()), m.total)
|
|
121
|
+
return (
|
|
122
|
+
f'Last {m.days} days: {wasted} of {m.total.turns} turns wasted, '
|
|
123
|
+
f'{rebuilt} more rebuilt an expired cache and {long} ran past half their context: {share:.1%} of your usage.'
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _measurement_header(*, price: float | None) -> list[str]:
|
|
128
|
+
return ['Kind', 'Turns', 'Share of usage', 'Tokens'] + (['€'] if price is not None else [])
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _measurement_rows(m: Measurement, *, price: float | None) -> list[list[str]]:
|
|
132
|
+
return [
|
|
133
|
+
[_KIND_LABELS[kind], str(t.turns), f'{_share(t.weight, m.total):.1%}', f'{t.tokens:,}']
|
|
134
|
+
+ ([f'€{_eur(t, price=price):.2f}'] if price is not None else [])
|
|
135
|
+
for kind, t in sorted(m.waste.items(), key=lambda kv: -kv[1].tokens)
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def measurement_json(m: Measurement, *, price: float | None) -> str:
|
|
140
|
+
"""Returns the measurement as JSON, with `eur` only when `price` is set."""
|
|
141
|
+
doc = {
|
|
142
|
+
'report': 'measure',
|
|
143
|
+
'days': m.days,
|
|
144
|
+
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
145
|
+
} | _measurement_doc(m, price=price)
|
|
146
|
+
return json.dumps(doc, ensure_ascii=False, indent=2)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _measurement_doc(m: Measurement, *, price: float | None) -> dict:
|
|
150
|
+
def tally(t: Tally) -> dict:
|
|
151
|
+
return dataclasses.asdict(t) | ({'eur': round(_eur(t, price=price), 2)} if price is not None else {})
|
|
152
|
+
|
|
153
|
+
return {
|
|
154
|
+
'total': tally(m.total),
|
|
155
|
+
'waste': {kind: tally(t) | {'share': _share(t.weight, m.total)} for kind, t in m.waste.items()},
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def audit_json(a: Audit) -> str:
|
|
160
|
+
"""Returns the audit as JSON for the skill: measurement, findings with their cost, and settings checks."""
|
|
161
|
+
doc = {
|
|
162
|
+
'report': 'audit',
|
|
163
|
+
'days': a.measurement.days,
|
|
164
|
+
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
165
|
+
'judge': a.judge,
|
|
166
|
+
'note': a.note,
|
|
167
|
+
} | _measurement_doc(a.measurement, price=None) | {
|
|
168
|
+
'findings': [finding_dict(f) for f in a.findings],
|
|
169
|
+
'checks': [dataclasses.asdict(c) for c in a.checks],
|
|
170
|
+
}
|
|
171
|
+
return json.dumps(doc, ensure_ascii=False, indent=2)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def audit_markdown(a: Audit, *, top: int) -> str:
|
|
175
|
+
"""Returns the audit as markdown: the headline, the costliest `top` findings, then failing checks."""
|
|
176
|
+
lines = ['# Audit', '', f'_{_measurement_headline(a.measurement)}_', '', a.note]
|
|
177
|
+
wasted = [f for f in a.findings if f.cost is not None]
|
|
178
|
+
for i, f in enumerate(wasted[:top], start=1):
|
|
179
|
+
lines.append(f'{i}. **{f.share:.1%}** {f.kind}, {f.cost.turns} turns: {_shown(f.variants[0].text)}')
|
|
180
|
+
lines += ['', _checks_headline(a.checks)] + [f'- {c.name}: {c.fix}' for c in a.checks if not c.ok]
|
|
181
|
+
return '\n'.join(lines)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _share(weight: float, total: Tally) -> float:
|
|
185
|
+
return weight / total.weight if total.weight else 0.0
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _eur(t: Tally, *, price: float) -> float:
|
|
189
|
+
return t.weight / 1e6 * price
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def checks_markdown(checks: list[Check]) -> str:
|
|
193
|
+
"""Returns the doctor's checks as a markdown table."""
|
|
194
|
+
lines = ['# Settings doctor', '', f'_{_checks_headline(checks)}_', '', '| Check | OK | Found | Fix |', '|---|---|---|---|']
|
|
195
|
+
lines += ['| ' + ' | '.join(row) + ' |' for row in _checks_rows(checks)]
|
|
196
|
+
return '\n'.join(lines)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def checks_table(checks: list[Check]) -> str:
|
|
200
|
+
"""Returns the doctor's checks as a rich table for a terminal."""
|
|
201
|
+
from rich.console import Console
|
|
202
|
+
from rich.table import Table
|
|
203
|
+
|
|
204
|
+
table = Table(title='Settings doctor', caption=_checks_headline(checks))
|
|
205
|
+
for column in ('Check', 'OK', 'Found', 'Fix'):
|
|
206
|
+
table.add_column(column)
|
|
207
|
+
for row in _checks_rows(checks):
|
|
208
|
+
table.add_row(*row)
|
|
209
|
+
console = Console()
|
|
210
|
+
with console.capture() as captured:
|
|
211
|
+
console.print(table)
|
|
212
|
+
return captured.get()
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def checks_json(checks: list[Check]) -> str:
|
|
216
|
+
"""Returns the doctor's checks as JSON."""
|
|
217
|
+
doc = {
|
|
218
|
+
'report': 'doctor',
|
|
219
|
+
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
220
|
+
'checks': [dataclasses.asdict(c) for c in checks],
|
|
221
|
+
}
|
|
222
|
+
return json.dumps(doc, ensure_ascii=False, indent=2)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _checks_headline(checks: list[Check]) -> str:
|
|
226
|
+
return f'{sum(not c.ok for c in checks)} of {len(checks)} checks to fix.'
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _checks_rows(checks: list[Check]) -> list[list[str]]:
|
|
230
|
+
return [[_CHECK_LABELS[c.name], '✓' if c.ok else '✗', c.found, c.fix] for c in checks]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _section(heading: str, found: list[Finding], *, days: int, note: str, summary: str, top: int) -> str:
|
|
234
|
+
lines = [heading, '', f'_Last {days} days: {summary}_ {note}'.rstrip(), '']
|
|
235
|
+
for i, f in enumerate(found[:top], start=1):
|
|
236
|
+
variants = ' · '.join(f'{v.count} × {_shown(v.text)}' for v in f.variants[:_SHOWN_VARIANTS])
|
|
237
|
+
lines.append(f'{i}. **{f.total}**: {variants}')
|
|
238
|
+
return '\n'.join(lines)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _asks(report: Report, *, top: int) -> str:
|
|
242
|
+
t = report.totals
|
|
243
|
+
lines = [
|
|
244
|
+
'# What Claude asks that always gets a yes',
|
|
245
|
+
'',
|
|
246
|
+
f'_Last {report.days} days: {t["wasted"]} of {t["turn_endings"]} turn endings were answered with a bare '
|
|
247
|
+
f'go-ahead ({t["not_questions"]} of them not even phrased as a question)._',
|
|
248
|
+
'',
|
|
249
|
+
report.note,
|
|
250
|
+
]
|
|
251
|
+
for f in report.findings:
|
|
252
|
+
lines += ['', f'## {f.label}: {f.total}' if f.label else '## Closing lines', '']
|
|
253
|
+
lines += [f'- {v.count} × {_shown(v.text)}' for v in f.variants[:top]]
|
|
254
|
+
return '\n'.join(lines)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _shown(text: str) -> str:
|
|
258
|
+
text = re.sub(r'\s+', ' ', text).strip()
|
|
259
|
+
return text[:_MAX_SHOWN_CHARS] + '…' if len(text) > _MAX_SHOWN_CHARS else text
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: repeat-o-scope
|
|
3
|
+
description: Use when the user asks where their Claude Code usage goes, how to make it cost less, what they keep repeating, wants to "automate the boring things", asks which prompts, commands, approvals or tool failures recur, or runs /repeat-o-scope. Audits the last 30 days of transcripts and settings in one pass and turns the costliest causes into concrete fixes the user approves.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# repeat-o-scope
|
|
7
|
+
|
|
8
|
+
Measure what the user's Claude Code sessions waste, find the common causes, and apply only the fixes the user picks, costliest first.
|
|
9
|
+
|
|
10
|
+
## 1. Check the tool
|
|
11
|
+
|
|
12
|
+
Run `repeat-o-scope --help`. If the command is not found, tell the user to install it with `uv tool install repeat-o-scope` and then `repeat-o-scope install`, and stop.
|
|
13
|
+
|
|
14
|
+
## 2. Measure
|
|
15
|
+
|
|
16
|
+
Run this and parse the JSON from stdout (progress goes to stderr):
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
repeat-o-scope audit --format json --jev
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`--jev` judges the findings with the first judge whose key is set and that answers a test call: Jev, Gemini Decision 1, then Clef. Never read, look up or ask for a secret yourself. Add `--all` only when the user asked to see everything, including findings they declined before.
|
|
23
|
+
|
|
24
|
+
The JSON has:
|
|
25
|
+
- `total` and `waste`: usage per kind, each with its `share` of all usage. `cache_rebuild` and `long_context` have no findings; failing `checks` explain them.
|
|
26
|
+
- `findings`: waste findings first, costliest first. Each has an `id`, a `kind` (`refused_edit`, `reread`, `failed`, `denied`, `bare_yes`), a `total`, its `variants`, redacted `examples`, a `cost` and a `share`. Then `prompt` and `command` findings, which have only a `total`.
|
|
27
|
+
- `judge`: which judge ran, or `null`. `label` is `real` for a failed group the judge kept, and the kind of ask for a `bare_yes` group.
|
|
28
|
+
- `checks`: the settings checks. Each has `ok`, what was `found`, and a `fix`.
|
|
29
|
+
|
|
30
|
+
## 3. Judge, when no judge ran
|
|
31
|
+
|
|
32
|
+
If `judge` is `null`, spawn one subagent with `model: "haiku"`. Give it the top example of each `failed` and `bare_yes` finding, and these questions. Its prompt starts with: "The Bash tool does not exist here. Use PowerShell for commands and Read/Grep/Glob for files."
|
|
33
|
+
|
|
34
|
+
- For each `failed` example, yes or no: An AI coding assistant made a tool call, and the result was flagged as an error. `failure` reads "<tool>: <what it tried> → <first line of the error>". Did the attempt really fail to do what it was meant to do? Answer no when the error is only normal output written to stderr, a warning, a progress line, or a non-zero exit the command was expected to produce, such as a test run or a linter reporting what it found.
|
|
35
|
+
- For each `bare_yes` example, one of these kinds. The line is the last line of an AI coding assistant's turn, and the user answered it only with a go-ahead such as "ship", "go" or "yes":
|
|
36
|
+
- `permission`: Asks for permission or a go-ahead for a next step the assistant could have taken by itself: push, commit, ship, post, run, continue. Includes "say "ship" and I'll …", "reply go and …", "want me to …?".
|
|
37
|
+
- `choice`: Asks the user to pick between real alternatives that change what gets built or done.
|
|
38
|
+
- `clarifying`: Asks for a fact or a preference that only the user knows.
|
|
39
|
+
- `review`: Asks the user to look at and approve something shown: a design, a draft, a diff, a plan.
|
|
40
|
+
- `report`: Asks for nothing: states a result, a status, a caveat or a list item.
|
|
41
|
+
|
|
42
|
+
Drop the `failed` findings it answers no for, and use its kinds as the `bare_yes` labels.
|
|
43
|
+
|
|
44
|
+
## 4. Find the themes
|
|
45
|
+
|
|
46
|
+
Merge findings of any kind that share one root cause into a theme, for example "Windows shell confusion": denied Bash, failed quoting, and the retries after them. A theme's share is the sum of its findings' shares. A failing check joins the theme it explains (`idle_compact` with expired caches, `skills` with fixed context per turn), or is a theme of its own.
|
|
47
|
+
|
|
48
|
+
`bare_yes` findings labelled `permission` are turns where Claude asked for a go-ahead it didn't need. The fix is a rule that stops the asking, not a faster way to say yes.
|
|
49
|
+
|
|
50
|
+
## 5. Plan at most 4 fixes
|
|
51
|
+
|
|
52
|
+
Take the 4 themes with the largest share. `prompt` and `command` findings compete, by `total`, only when there are fewer than 4 waste themes. If there is nothing, say so in one line and stop.
|
|
53
|
+
|
|
54
|
+
For each theme, read the files a fix would touch before drafting it:
|
|
55
|
+
|
|
56
|
+
- the user's shell profile (`$PROFILE` in PowerShell, `~/.zshrc` or `~/.bashrc`)
|
|
57
|
+
- `~/.claude/settings.json` and `~/.claude/CLAUDE.md`
|
|
58
|
+
- the skill, hook or script the theme points at
|
|
59
|
+
|
|
60
|
+
Then draft one to three **concrete** fixes. Concrete means the exact text that would be added or changed and the file it goes in, never advice such as "consider automating this". Fix kinds:
|
|
61
|
+
|
|
62
|
+
- a line in `~/.claude/CLAUDE.md` for a repeated correction or a repeated failure (Edit before Read, wrong shell syntax)
|
|
63
|
+
- a permission rule, a setting or an env var in `~/.claude/settings.json`
|
|
64
|
+
- a Claude Code hook in `~/.claude/settings.json` (a SessionStart credential check, a Stop or PreToolUse rule)
|
|
65
|
+
- a shell function, alias or script for a repeated command
|
|
66
|
+
- an edit to a skill's text, or a fix to a script the user owns (a `/sync` that keeps hitting conflicts)
|
|
67
|
+
- for too many skills: add `disable-model-invocation: true` to the frontmatter of one of the user's own skills in `~/.claude/skills`, so it runs only as `/name` and costs nothing per turn; or switch a plugin skill off with `skillOverrides` or `enabledPlugins`. Never edit a skill in the plugin cache: an update overwrites it.
|
|
68
|
+
|
|
69
|
+
## 6. Ask once
|
|
70
|
+
|
|
71
|
+
Make one `AskUserQuestion` call with one question per theme. In each question, give the theme in a few words with its share of usage, and list each fix as an option whose label says what "yes" does, for example "Append the Edit-before-Read line to ~/.claude/CLAUDE.md". Every question also gets the option `Leave it`.
|
|
72
|
+
|
|
73
|
+
## 7. Apply
|
|
74
|
+
|
|
75
|
+
- For each picked fix, make exactly that change and nothing else.
|
|
76
|
+
- For each `Leave it`, run `repeat-o-scope decline <id>` for every finding in that theme so it isn't suggested again.
|
|
77
|
+
- Report what changed as one short list: file and change. Then one line with the wasted share from the headline, and that `audit` saved today's measurement as the baseline to compare against later.
|
|
78
|
+
- If more themes are left and the user approved at least one fix, ask once whether to do the next batch. Otherwise end with the list. Don't ask "want me to…?" about anything already picked.
|
|
79
|
+
|
|
80
|
+
## Never
|
|
81
|
+
|
|
82
|
+
- Push, commit, post, or file issues.
|
|
83
|
+
- Change a file the picked option didn't name.
|
|
84
|
+
- Send findings anywhere other than the judges `audit` itself uses: they quote the user's own prompts.
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Reads Claude Code transcripts: what the user typed, how Claude ended turns, which tool calls failed."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from repeat_o_scope.redaction import redact
|
|
12
|
+
|
|
13
|
+
_LOGGER = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
# Harness plumbing that arrives as a user message but was not typed by the user.
|
|
16
|
+
_PLUMBING = re.compile(
|
|
17
|
+
r'^\s*(<(command-|local-command|system-reminder|task-notification|cross-session-message|pasted_content'
|
|
18
|
+
r'|bash-stdout|bash-stderr)|This session is being continued|\[Request interrupted|Base directory for this skill'
|
|
19
|
+
r'|Caveat:)',
|
|
20
|
+
re.IGNORECASE,
|
|
21
|
+
)
|
|
22
|
+
# A `!` command the user ran by hand.
|
|
23
|
+
_BASH_INPUT = re.compile(r'^\s*<bash-input>(.*?)</bash-input>', re.IGNORECASE | re.DOTALL)
|
|
24
|
+
# Lines that wrap a tool error without saying anything about it.
|
|
25
|
+
_ERROR_WRAPPER = re.compile(r'^(</?tool_use_error>|</?error>|Exit code -?\d+)$')
|
|
26
|
+
_ATTEMPT_KEYS = ('command', 'file_path', 'pattern', 'query', 'skill', 'url', 'prompt')
|
|
27
|
+
# A reply that only says "go on": the turn before it asked for nothing that needed asking.
|
|
28
|
+
_GO_AHEAD = re.compile(
|
|
29
|
+
r'^(ship|ship it|go|go ahead|go for it|yes|yes please|yes, please|y|ok|okay|do it|do that|push|push it'
|
|
30
|
+
r'|commit|commit it|commit and push|commit, push and open mr|post|post it|post them|continue|proceed|sure'
|
|
31
|
+
r'|yep|please do)[.!]*$'
|
|
32
|
+
)
|
|
33
|
+
# Tool calls refused by a person, a hook or the permission classifier: not failures to automate away.
|
|
34
|
+
_DENIED = re.compile(
|
|
35
|
+
r"permission for this action was denied|doesn't want to proceed|denied this in vjuu|hook error|blocked by"
|
|
36
|
+
# A tool the settings deny or don't offer at all, e.g. Bash on Windows.
|
|
37
|
+
r'|no such tool available|permission to use \w+ has been denied',
|
|
38
|
+
re.IGNORECASE,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class TurnEnd:
|
|
44
|
+
closing: str
|
|
45
|
+
reply: str
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class ToolFailure:
|
|
50
|
+
tool: str
|
|
51
|
+
attempt: str
|
|
52
|
+
error: str
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def iter_prompts(*, root: Path, since: datetime) -> Iterator[str]:
|
|
56
|
+
"""Yields the prompts the user typed, redacted."""
|
|
57
|
+
for text in _iter_typed(root=root, since=since):
|
|
58
|
+
if not _BASH_INPUT.match(text):
|
|
59
|
+
yield redact(text)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def iter_commands(*, root: Path, since: datetime) -> Iterator[str]:
|
|
63
|
+
"""Yields the `!` shell commands the user ran by hand, redacted."""
|
|
64
|
+
for text in _iter_typed(root=root, since=since):
|
|
65
|
+
if match := _BASH_INPUT.match(text):
|
|
66
|
+
yield redact(match.group(1).strip())
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def iter_turn_ends(*, root: Path, since: datetime) -> Iterator[TurnEnd]:
|
|
70
|
+
"""Yields Claude's closing line of each turn together with the user's reply."""
|
|
71
|
+
closing = {}
|
|
72
|
+
for e in iter_entries(root=root, since=since, subagents=False):
|
|
73
|
+
session = e.get('sessionId')
|
|
74
|
+
if e.get('type') == 'assistant' and not e.get('isSidechain'):
|
|
75
|
+
if line := last_line(message_text(e)):
|
|
76
|
+
closing[session] = line
|
|
77
|
+
elif (text := typed_text(e)) and not _BASH_INPUT.match(text) and session in closing:
|
|
78
|
+
yield TurnEnd(closing=redact(closing.pop(session)), reply=redact(text))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def iter_tool_failures(*, root: Path, since: datetime) -> Iterator[ToolFailure]:
|
|
82
|
+
"""Yields every failed tool call, subagents included, with what was attempted."""
|
|
83
|
+
attempts = {}
|
|
84
|
+
for e in iter_entries(root=root, since=since, subagents=True):
|
|
85
|
+
content = (e.get('message') or {}).get('content')
|
|
86
|
+
if not isinstance(content, list):
|
|
87
|
+
continue
|
|
88
|
+
for block in content:
|
|
89
|
+
if not isinstance(block, dict):
|
|
90
|
+
continue
|
|
91
|
+
if block.get('type') == 'tool_use':
|
|
92
|
+
attempts[block.get('id')] = (block.get('name') or '?', attempt(block.get('input')))
|
|
93
|
+
elif block.get('type') == 'tool_result' and block.get('is_error'):
|
|
94
|
+
tool, tried = attempts.get(block.get('tool_use_id'), ('?', ''))
|
|
95
|
+
yield ToolFailure(tool=tool, attempt=redact(tried), error=redact(error_line(block.get('content'))))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def is_go_ahead(reply: str) -> bool:
|
|
99
|
+
"""Returns whether a reply only says "go on"."""
|
|
100
|
+
return bool(_GO_AHEAD.match(re.sub(r'\s+', ' ', reply.lower().strip())))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def is_denial(error: str) -> bool:
|
|
104
|
+
"""Returns whether a tool error says a person, a hook or the permission classifier refused the call."""
|
|
105
|
+
return bool(_DENIED.search(error))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def iter_entries(*, root: Path, since: datetime, subagents: bool) -> Iterator[dict]:
|
|
109
|
+
"""Yields transcript entries newer than `since`, skipping lines that are not JSON objects."""
|
|
110
|
+
files = 0
|
|
111
|
+
for path in sorted(root.rglob('*.jsonl')):
|
|
112
|
+
if path.stat().st_mtime < since.timestamp() or (not subagents and 'subagents' in path.parts):
|
|
113
|
+
continue
|
|
114
|
+
files += 1
|
|
115
|
+
with path.open(encoding='utf-8', errors='replace') as f:
|
|
116
|
+
for line in f:
|
|
117
|
+
try:
|
|
118
|
+
e = json.loads(line)
|
|
119
|
+
except json.JSONDecodeError:
|
|
120
|
+
continue
|
|
121
|
+
if isinstance(e, dict) and _newer(e.get('timestamp'), since):
|
|
122
|
+
yield e
|
|
123
|
+
_LOGGER.info('Read `%s` transcripts from `%s`.', files, root)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _iter_typed(*, root: Path, since: datetime) -> Iterator[str]:
|
|
127
|
+
for e in iter_entries(root=root, since=since, subagents=False):
|
|
128
|
+
if text := typed_text(e):
|
|
129
|
+
yield text
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def typed_text(e: dict) -> str | None:
|
|
133
|
+
"""Returns the text of a message the user typed, or None for anything else."""
|
|
134
|
+
if (
|
|
135
|
+
e.get('type') != 'user'
|
|
136
|
+
or e.get('isMeta')
|
|
137
|
+
or e.get('isSidechain')
|
|
138
|
+
# Scripted runs (`claude -p`, the Agent SDK) are not the user typing.
|
|
139
|
+
or e.get('entrypoint') == 'sdk-cli'
|
|
140
|
+
):
|
|
141
|
+
return None
|
|
142
|
+
text = message_text(e).strip()
|
|
143
|
+
return text if text and not _PLUMBING.match(text) else None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def message_text(e: dict) -> str:
|
|
147
|
+
"""Returns the text blocks of an entry's message, joined."""
|
|
148
|
+
content = (e.get('message') or {}).get('content')
|
|
149
|
+
if isinstance(content, str):
|
|
150
|
+
return content
|
|
151
|
+
if not isinstance(content, list):
|
|
152
|
+
return ''
|
|
153
|
+
return ' '.join(b.get('text', '') for b in content if isinstance(b, dict) and b.get('type') == 'text')
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def last_line(text: str) -> str:
|
|
157
|
+
"""Returns the last line of `text` that isn't blank."""
|
|
158
|
+
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
159
|
+
return lines[-1] if lines else ''
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def attempt(tool_input) -> str:
|
|
163
|
+
"""Returns what a tool call tried to do: its command, path or query."""
|
|
164
|
+
if not isinstance(tool_input, dict):
|
|
165
|
+
return ''
|
|
166
|
+
for key in _ATTEMPT_KEYS:
|
|
167
|
+
if key in tool_input:
|
|
168
|
+
return str(tool_input[key])
|
|
169
|
+
return json.dumps(tool_input)[:200]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def error_line(content) -> str:
|
|
173
|
+
"""Returns the first line of a tool error that says something."""
|
|
174
|
+
text = content if isinstance(content, str) else ' '.join(
|
|
175
|
+
b.get('text', '') for b in content or [] if isinstance(b, dict) and b.get('type') == 'text'
|
|
176
|
+
)
|
|
177
|
+
for line in text.splitlines():
|
|
178
|
+
line = line.strip()
|
|
179
|
+
if line and not _ERROR_WRAPPER.match(line):
|
|
180
|
+
return re.sub(r'</?tool_use_error>', '', line).strip()
|
|
181
|
+
return ''
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _newer(timestamp, since: datetime) -> bool:
|
|
185
|
+
if not isinstance(timestamp, str):
|
|
186
|
+
return False
|
|
187
|
+
try:
|
|
188
|
+
return datetime.fromisoformat(timestamp) >= since
|
|
189
|
+
except ValueError:
|
|
190
|
+
return False
|