repeat-o-scope 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repeat_o_scope/__init__.py +0 -0
- repeat_o_scope/audit.py +136 -0
- repeat_o_scope/cli.py +248 -0
- repeat_o_scope/cluster.py +76 -0
- repeat_o_scope/declined.py +70 -0
- repeat_o_scope/doctor.py +211 -0
- repeat_o_scope/findings.py +71 -0
- repeat_o_scope/install.py +103 -0
- repeat_o_scope/judge.py +67 -0
- repeat_o_scope/judges.py +218 -0
- repeat_o_scope/redaction.py +16 -0
- repeat_o_scope/render.py +259 -0
- repeat_o_scope/skill/SKILL.md +84 -0
- repeat_o_scope/transcripts.py +190 -0
- repeat_o_scope/waste.py +262 -0
- repeat_o_scope/worklog.py +233 -0
- repeat_o_scope-0.1.0.dist-info/METADATA +255 -0
- repeat_o_scope-0.1.0.dist-info/RECORD +21 -0
- repeat_o_scope-0.1.0.dist-info/WHEEL +4 -0
- repeat_o_scope-0.1.0.dist-info/entry_points.txt +4 -0
- repeat_o_scope-0.1.0.dist-info/licenses/LICENSE +21 -0
|
File without changes
|
repeat_o_scope/audit.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""One pass over the transcripts and settings: what was wasted, on what, what it cost, and what to check."""
|
|
2
|
+
|
|
3
|
+
import collections
|
|
4
|
+
import dataclasses
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from repeat_o_scope import cluster, doctor, findings, judge, judges, transcripts, waste
|
|
12
|
+
from repeat_o_scope.findings import Finding, Report, Variant
|
|
13
|
+
from repeat_o_scope.waste import Charge, Measurement, Tally
|
|
14
|
+
|
|
15
|
+
_LOGGER = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
_MIN_SIZE = 4
|
|
18
|
+
# Wasted calls cost something even once, so two alike already make a group.
|
|
19
|
+
_MIN_WASTE_SIZE = 2
|
|
20
|
+
_MAX_EXAMPLES = 3
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Audit:
|
|
25
|
+
measurement: Measurement
|
|
26
|
+
judge: str | None
|
|
27
|
+
findings: list[Finding]
|
|
28
|
+
checks: list[doctor.Check]
|
|
29
|
+
note: str = ''
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run(*, claude_dir: Path, days: int, judge: bool, project_dir: Path, root: Path, since: datetime) -> Audit:
|
|
33
|
+
"""Returns the audit: waste findings ranked by share of usage, then prompts and commands, then settings."""
|
|
34
|
+
m = waste.measure(days=days, root=root, since=since)
|
|
35
|
+
wasted = [f for kind in waste.KINDS for f in _waste_findings(m, kind=kind)]
|
|
36
|
+
opened = judges.open_judge() if judge else None
|
|
37
|
+
note = ''
|
|
38
|
+
if opened is not None:
|
|
39
|
+
wasted, note = _judged(wasted, by=opened)
|
|
40
|
+
wasted.sort(key=lambda f: -f.share)
|
|
41
|
+
counted = [
|
|
42
|
+
*clustered('prompts', 'prompt', transcripts.iter_prompts(root=root, since=since), days=days).findings,
|
|
43
|
+
*clustered('commands', 'command', transcripts.iter_commands(root=root, since=since), days=days).findings,
|
|
44
|
+
]
|
|
45
|
+
checks = doctor.diagnose(claude_dir=claude_dir, project_dir=project_dir)
|
|
46
|
+
_LOGGER.info('Audited `%s` waste findings and `%s` repeats.', len(wasted), len(counted))
|
|
47
|
+
return Audit(
|
|
48
|
+
measurement=m,
|
|
49
|
+
judge=opened.name if opened else None,
|
|
50
|
+
findings=wasted + counted,
|
|
51
|
+
checks=checks,
|
|
52
|
+
note=note,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def clustered(name: str, kind: str, texts: Iterable[str], *, days: int, jev: bool = False) -> Report:
|
|
57
|
+
"""Returns a report of the texts that repeat at least `_MIN_SIZE` times, grouped."""
|
|
58
|
+
originals = collections.defaultdict(list)
|
|
59
|
+
for text in texts:
|
|
60
|
+
originals[cluster.normalise(text)].append(text)
|
|
61
|
+
clusters, noise = cluster.rank(repeated(originals), min_size=_MIN_SIZE)
|
|
62
|
+
_LOGGER.info('Ranked `%s` %s into `%s` groups.', sum(map(len, originals.values())), name, len(clusters))
|
|
63
|
+
return Report(
|
|
64
|
+
name=name,
|
|
65
|
+
days=days,
|
|
66
|
+
jev=jev,
|
|
67
|
+
totals={'items': sum(map(len, originals.values())), 'groups': len(clusters), 'one_offs': noise},
|
|
68
|
+
findings=[findings.from_cluster(report=name, kind=kind, cluster=c, originals=originals) for c in clusters],
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def repeated(groups: dict[str, list]) -> list[str]:
|
|
73
|
+
"""Returns each key once per item behind it, the shape `cluster.rank` counts."""
|
|
74
|
+
return [k for k, items in groups.items() for _ in items]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _waste_findings(m: Measurement, *, kind: str) -> list[Finding]:
|
|
78
|
+
"""Returns one finding per group of alike wasted calls of `kind`, and one for the rest."""
|
|
79
|
+
behind: dict[str, list[Charge]] = collections.defaultdict(list)
|
|
80
|
+
for c in m.charges:
|
|
81
|
+
if c.kind == kind:
|
|
82
|
+
# A re-read's evidence is its path, which normalising would turn into `<path>` for every file.
|
|
83
|
+
behind[c.evidence if kind == 'reread' else cluster.normalise(c.evidence)].append(c)
|
|
84
|
+
if not behind:
|
|
85
|
+
return []
|
|
86
|
+
if kind == 'reread':
|
|
87
|
+
# The same file, not a similar one: no embeddings.
|
|
88
|
+
clusters = [
|
|
89
|
+
cluster.Cluster(total=len(cs), variants=[(path, len(cs))])
|
|
90
|
+
for path, cs in sorted(behind.items(), key=lambda kv: (-len(kv[1]), kv[0]))
|
|
91
|
+
if len(cs) >= _MIN_WASTE_SIZE
|
|
92
|
+
]
|
|
93
|
+
else:
|
|
94
|
+
clusters, _ = cluster.rank(repeated(behind), min_size=_MIN_WASTE_SIZE)
|
|
95
|
+
grouped = {text for c in clusters for text, _ in c.variants}
|
|
96
|
+
rest = sorted(((t, len(cs)) for t, cs in behind.items() if t not in grouped), key=lambda v: (-v[1], v[0]))
|
|
97
|
+
groups = [(c.variants, c.variants[0][0]) for c in clusters]
|
|
98
|
+
if rest:
|
|
99
|
+
groups.append((rest, '*'))
|
|
100
|
+
return [_finding(kind=kind, variants=variants, top=top, behind=behind, total=m.total) for variants, top in groups]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _finding(*, behind: dict[str, list[Charge]], kind: str, top: str, total: Tally, variants: list) -> Finding:
|
|
104
|
+
charges = [c for text, _ in variants for c in behind[text]]
|
|
105
|
+
cost = Tally(
|
|
106
|
+
turns=sum(c.tally.turns for c in charges),
|
|
107
|
+
tokens=sum(c.tally.tokens for c in charges),
|
|
108
|
+
weight=sum(c.tally.weight for c in charges),
|
|
109
|
+
)
|
|
110
|
+
return Finding(
|
|
111
|
+
id=findings.finding_id(report='audit', top_variant=f'{kind}:{top}'),
|
|
112
|
+
kind=kind,
|
|
113
|
+
total=len(charges),
|
|
114
|
+
variants=[Variant(text, count) for text, count in variants],
|
|
115
|
+
examples=list(dict.fromkeys(c.evidence for c in behind[variants[0][0]]))[:_MAX_EXAMPLES],
|
|
116
|
+
label=None,
|
|
117
|
+
cost=cost,
|
|
118
|
+
share=cost.weight / total.weight if total.weight else 0.0,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _judged(found: list[Finding], *, by: judges.Judge) -> tuple[list[Finding], str]:
|
|
123
|
+
"""Returns the findings with failed-call noise dropped and go-aheads labelled, and a note on what was dropped."""
|
|
124
|
+
# A rest group of several texts mixes unrelated calls, so one example can't speak for it: keep it unlabelled.
|
|
125
|
+
rest = {findings.finding_id(report='audit', top_variant=f'{kind}:*') for kind in waste.KINDS}
|
|
126
|
+
judged = [f for f in found if f.id not in rest or len(f.variants) == 1]
|
|
127
|
+
failed = [f for f in judged if f.kind == 'failed']
|
|
128
|
+
asks = [f for f in judged if f.kind == 'bare_yes']
|
|
129
|
+
real = judge.real_failures([f.examples[0] for f in failed], judge=by) if failed else []
|
|
130
|
+
kinds = judge.kinds_of_asks([f.examples[0] for f in asks], judge=by) if asks else []
|
|
131
|
+
labels = {f.id: 'real' for f, r in zip(failed, real) if r} | {f.id: k for f, k in zip(asks, kinds)}
|
|
132
|
+
noise = {f.id for f, r in zip(failed, real) if not r}
|
|
133
|
+
kept = [dataclasses.replace(f, label=labels.get(f.id)) for f in found if f.id not in noise]
|
|
134
|
+
note = f'{by.name} judged {len(noise)} group(s) of failed calls as noise on stderr and left them out.'
|
|
135
|
+
return kept, note if failed else ''
|
|
136
|
+
|
repeat_o_scope/cli.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""repeat-o-scope: reports on the last N days of Claude Code transcripts."""
|
|
2
|
+
|
|
3
|
+
import collections
|
|
4
|
+
import dataclasses
|
|
5
|
+
import logging
|
|
6
|
+
from datetime import datetime, timedelta, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from repeat_o_scope import audit, cluster, declined, doctor, findings, install, judge, judges, render, transcripts, waste
|
|
10
|
+
from repeat_o_scope.findings import Finding, Report, Variant
|
|
11
|
+
|
|
12
|
+
_LOGGER = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
_MIN_SIZE = 4
|
|
15
|
+
_MAX_EXAMPLES = 3
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Scope:
|
|
19
|
+
"""Finds what keeps repeating in your Claude Code sessions."""
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
all: bool = False,
|
|
24
|
+
claude_dir: str | None = None,
|
|
25
|
+
data_dir: str | None = None,
|
|
26
|
+
days: int = 30,
|
|
27
|
+
format: str | None = None,
|
|
28
|
+
jev: bool = False,
|
|
29
|
+
price: float | None = None,
|
|
30
|
+
root: str | None = None,
|
|
31
|
+
top: int = 20,
|
|
32
|
+
):
|
|
33
|
+
"""`jev` sends closing lines and tool errors to TypeSafe (costs money, needs `TYPESAFE_API_KEY`).
|
|
34
|
+
|
|
35
|
+
`all` shows findings you declined, too.
|
|
36
|
+
"""
|
|
37
|
+
self._all = all
|
|
38
|
+
self._claude_dir = Path(claude_dir) if claude_dir else Path.home() / '.claude'
|
|
39
|
+
self._data_dir = Path(data_dir) if data_dir else declined.default_dir()
|
|
40
|
+
self._root = Path(root) if root else self._claude_dir / 'projects'
|
|
41
|
+
self._since = datetime.now(timezone.utc) - timedelta(days=days)
|
|
42
|
+
self._days = days
|
|
43
|
+
self._format = format
|
|
44
|
+
self._jev = jev
|
|
45
|
+
self._price = price
|
|
46
|
+
self._top = top
|
|
47
|
+
|
|
48
|
+
def prompts(self) -> str:
|
|
49
|
+
"""Prompts you keep typing."""
|
|
50
|
+
return self._out(self._report_prompts)
|
|
51
|
+
|
|
52
|
+
def commands(self) -> str:
|
|
53
|
+
"""Shell commands you keep running by hand with `!`."""
|
|
54
|
+
return self._out(self._report_commands)
|
|
55
|
+
|
|
56
|
+
def asks(self) -> str:
|
|
57
|
+
"""Turn endings that were answered with a bare go-ahead."""
|
|
58
|
+
return self._out(self._report_asks)
|
|
59
|
+
|
|
60
|
+
def fails(self) -> str:
|
|
61
|
+
"""Tool calls that keep failing, with denials apart."""
|
|
62
|
+
return self._out(self._report_fails)
|
|
63
|
+
|
|
64
|
+
def measure(self) -> str:
|
|
65
|
+
"""Turns wasted per kind, and their share of your usage. `--price` adds € per million input tokens."""
|
|
66
|
+
# No --format means a person at a terminal; agents ask for markdown or json.
|
|
67
|
+
renderers = {
|
|
68
|
+
None: render.measurement_table,
|
|
69
|
+
'markdown': render.measurement_markdown,
|
|
70
|
+
'json': render.measurement_json,
|
|
71
|
+
}
|
|
72
|
+
if self._format not in renderers:
|
|
73
|
+
raise findings.RepeatOScopeError(f'Unknown format `{self._format}`, expected markdown or json!')
|
|
74
|
+
m = waste.measure(days=self._days, root=self._root, since=self._since)
|
|
75
|
+
return renderers[self._format](m, price=self._price)
|
|
76
|
+
|
|
77
|
+
def doctor(self) -> str:
|
|
78
|
+
"""Checks your settings and CLAUDE.md against the budget checklist in docs/why.md."""
|
|
79
|
+
# Not through `_out`: checks are not findings to remember or decline.
|
|
80
|
+
renderers = {None: render.checks_table, 'markdown': render.checks_markdown, 'json': render.checks_json}
|
|
81
|
+
if self._format not in renderers:
|
|
82
|
+
raise findings.RepeatOScopeError(f'Unknown format `{self._format}`, expected markdown or json!')
|
|
83
|
+
return renderers[self._format](doctor.diagnose(claude_dir=self._claude_dir, project_dir=Path.cwd()))
|
|
84
|
+
|
|
85
|
+
def audit(self) -> str:
|
|
86
|
+
"""Waste, what it was spent on and what it cost, repeats and settings in one pass, for the skill.
|
|
87
|
+
|
|
88
|
+
Writes the measurement as today's baseline. `--jev` judges the findings with the first available judge.
|
|
89
|
+
"""
|
|
90
|
+
# JSON by default: the skill reads this, a person reads `measure` or `--format markdown`.
|
|
91
|
+
renderers = {None: render.audit_json, 'json': render.audit_json, 'markdown': self._audit_markdown}
|
|
92
|
+
if self._format not in renderers:
|
|
93
|
+
raise findings.RepeatOScopeError(f'Unknown format `{self._format}`, expected markdown or json!')
|
|
94
|
+
a = audit.run(
|
|
95
|
+
claude_dir=self._claude_dir,
|
|
96
|
+
days=self._days,
|
|
97
|
+
judge=self._jev,
|
|
98
|
+
project_dir=Path.cwd(),
|
|
99
|
+
root=self._root,
|
|
100
|
+
since=self._since,
|
|
101
|
+
)
|
|
102
|
+
report = Report(name='audit', days=self._days, jev=self._jev, totals={}, findings=a.findings)
|
|
103
|
+
declined.remember_last(data_dir=self._data_dir, report=report)
|
|
104
|
+
if not self._all:
|
|
105
|
+
a = dataclasses.replace(a, findings=declined.drop_declined(data_dir=self._data_dir, findings=a.findings))
|
|
106
|
+
baseline = self._data_dir / 'baseline' / f'{datetime.now(timezone.utc).date().isoformat()}.json'
|
|
107
|
+
baseline.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
baseline.write_text(render.measurement_json(a.measurement, price=None), encoding='utf-8')
|
|
109
|
+
_LOGGER.info('Wrote the baseline to `%s`.', baseline)
|
|
110
|
+
return renderers[self._format](a)
|
|
111
|
+
|
|
112
|
+
def _audit_markdown(self, a: audit.Audit) -> str:
|
|
113
|
+
return render.audit_markdown(a, top=self._top)
|
|
114
|
+
|
|
115
|
+
def install(self) -> str:
|
|
116
|
+
"""Installs the repeat-o-scope skill and the worklog Stop hook into ~/.claude."""
|
|
117
|
+
return '\n'.join(install.install(claude_dir=self._claude_dir)) or 'Already installed.'
|
|
118
|
+
|
|
119
|
+
def uninstall(self) -> str:
|
|
120
|
+
"""Removes the repeat-o-scope skill and the worklog Stop hook from ~/.claude."""
|
|
121
|
+
return '\n'.join(install.uninstall(claude_dir=self._claude_dir)) or 'Nothing to remove.'
|
|
122
|
+
|
|
123
|
+
def decline(self, finding_id: str) -> str:
|
|
124
|
+
"""Leaves a finding from the last run out of future reports."""
|
|
125
|
+
variants = declined.decline(data_dir=self._data_dir, finding_id=finding_id)
|
|
126
|
+
return f'Declined `{finding_id}`: {", ".join(variants)}'
|
|
127
|
+
|
|
128
|
+
def _out(self, build) -> str:
|
|
129
|
+
renderers = {None: render.markdown, 'markdown': render.markdown, 'json': render.json_document}
|
|
130
|
+
if self._format not in renderers:
|
|
131
|
+
raise findings.RepeatOScopeError(f'Unknown format `{self._format}`, expected markdown or json!')
|
|
132
|
+
report = build()
|
|
133
|
+
declined.remember_last(data_dir=self._data_dir, report=report)
|
|
134
|
+
if not self._all:
|
|
135
|
+
report = dataclasses.replace(
|
|
136
|
+
report,
|
|
137
|
+
findings=declined.drop_declined(data_dir=self._data_dir, findings=report.findings),
|
|
138
|
+
)
|
|
139
|
+
return renderers[self._format](report, top=self._top)
|
|
140
|
+
|
|
141
|
+
def _report_prompts(self) -> Report:
|
|
142
|
+
return self._clustered('prompts', 'prompt', transcripts.iter_prompts(root=self._root, since=self._since))
|
|
143
|
+
|
|
144
|
+
def _report_commands(self) -> Report:
|
|
145
|
+
return self._clustered('commands', 'command', transcripts.iter_commands(root=self._root, since=self._since))
|
|
146
|
+
|
|
147
|
+
def _report_asks(self) -> Report:
|
|
148
|
+
ends = list(transcripts.iter_turn_ends(root=self._root, since=self._since))
|
|
149
|
+
wasted = [e for e in ends if transcripts.is_go_ahead(e.reply)]
|
|
150
|
+
replies = collections.Counter(e.reply.lower().strip() for e in wasted)
|
|
151
|
+
closings = collections.Counter(e.closing for e in wasted)
|
|
152
|
+
if self._jev and closings:
|
|
153
|
+
distinct = list(closings)
|
|
154
|
+
kind_of = dict(zip(distinct, judge.kinds_of_asks(distinct, judge=_open_judge())))
|
|
155
|
+
groups = collections.defaultdict(collections.Counter)
|
|
156
|
+
for c, n in closings.items():
|
|
157
|
+
groups[kind_of[c]][c] = n
|
|
158
|
+
else:
|
|
159
|
+
groups = {None: closings} if closings else {}
|
|
160
|
+
found = [
|
|
161
|
+
Finding(
|
|
162
|
+
# Not from the top closing line: exact phrasings rarely repeat, the kind of ask does.
|
|
163
|
+
id=findings.finding_id(report='asks', top_variant=f'*{label or ""}'),
|
|
164
|
+
kind='ask',
|
|
165
|
+
total=group.total(),
|
|
166
|
+
variants=[Variant(c, n) for c, n in group.most_common()],
|
|
167
|
+
examples=[c for c, _ in group.most_common(_MAX_EXAMPLES)],
|
|
168
|
+
label=label,
|
|
169
|
+
)
|
|
170
|
+
for label, group in sorted(groups.items(), key=lambda kv: -kv[1].total())
|
|
171
|
+
]
|
|
172
|
+
_LOGGER.info('Found `%s` wasted turn endings.', len(wasted))
|
|
173
|
+
return Report(
|
|
174
|
+
name='asks',
|
|
175
|
+
days=self._days,
|
|
176
|
+
jev=self._jev,
|
|
177
|
+
totals={
|
|
178
|
+
'turn_endings': len(ends),
|
|
179
|
+
'wasted': len(wasted),
|
|
180
|
+
'not_questions': sum('?' not in e.closing[-3:] for e in wasted),
|
|
181
|
+
},
|
|
182
|
+
findings=found,
|
|
183
|
+
note='Replies: ' + ', '.join(f'{n} × {r}' for r, n in replies.most_common(8)),
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
def _report_fails(self) -> Report:
|
|
187
|
+
failed, denied = collections.defaultdict(list), collections.defaultdict(list)
|
|
188
|
+
for f in transcripts.iter_tool_failures(root=self._root, since=self._since):
|
|
189
|
+
key = cluster.normalise(f'{f.tool}: {f.error}')
|
|
190
|
+
(denied if transcripts.is_denial(f.error) else failed)[key].append(f)
|
|
191
|
+
failure_clusters, failure_noise = cluster.rank(audit.repeated(failed), min_size=_MIN_SIZE)
|
|
192
|
+
note, label = '', None
|
|
193
|
+
if self._jev and failure_clusters:
|
|
194
|
+
# One example per group: the first failure behind its most frequent variant.
|
|
195
|
+
judged_by = _open_judge()
|
|
196
|
+
firsts = [failed[c.variants[0][0]][0] for c in failure_clusters]
|
|
197
|
+
real = judge.real_failures([f'{f.tool}: {f.attempt} → {f.error}' for f in firsts], judge=judged_by)
|
|
198
|
+
note = f'{judged_by.name} judged {sum(not r for r in real)} group(s) as noise on stderr and left them out.'
|
|
199
|
+
failure_clusters, label = [c for c, r in zip(failure_clusters, real) if r], 'real'
|
|
200
|
+
denial_clusters, denial_noise = cluster.rank(audit.repeated(denied), min_size=_MIN_SIZE)
|
|
201
|
+
originals = {
|
|
202
|
+
k: [f'{f.tool}: {f.attempt} → {f.error}' for f in fs] for k, fs in (failed | denied).items()
|
|
203
|
+
}
|
|
204
|
+
found = [
|
|
205
|
+
findings.from_cluster(report='fails', kind='failure', cluster=c, originals=originals, label=label)
|
|
206
|
+
for c in failure_clusters
|
|
207
|
+
] + [findings.from_cluster(report='fails', kind='denial', cluster=c, originals=originals) for c in denial_clusters]
|
|
208
|
+
return Report(
|
|
209
|
+
name='fails',
|
|
210
|
+
days=self._days,
|
|
211
|
+
jev=self._jev,
|
|
212
|
+
totals={
|
|
213
|
+
'items': sum(map(len, failed.values())),
|
|
214
|
+
'groups': len(failure_clusters),
|
|
215
|
+
'one_offs': failure_noise,
|
|
216
|
+
'denials': sum(map(len, denied.values())),
|
|
217
|
+
'denial_groups': len(denial_clusters),
|
|
218
|
+
'denial_one_offs': denial_noise,
|
|
219
|
+
},
|
|
220
|
+
findings=found,
|
|
221
|
+
note=note,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def _clustered(self, name: str, kind: str, texts) -> Report:
|
|
225
|
+
return audit.clustered(name, kind, texts, days=self._days, jev=self._jev)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _open_judge() -> judges.Judge:
|
|
229
|
+
if (opened := judges.open_judge()) is None:
|
|
230
|
+
raise findings.RepeatOScopeError('No judge available: set a Jev, Gemini or Cloudflare key!')
|
|
231
|
+
return opened
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def main() -> None:
|
|
235
|
+
import sys
|
|
236
|
+
|
|
237
|
+
import fire
|
|
238
|
+
from rich.console import Console
|
|
239
|
+
from rich.logging import RichHandler
|
|
240
|
+
|
|
241
|
+
# Findings quote prompts in any language; a legacy Windows code page must not crash the report.
|
|
242
|
+
sys.stdout.reconfigure(encoding='utf-8')
|
|
243
|
+
# The report goes to stdout so it can be piped or saved; progress goes to stderr.
|
|
244
|
+
handler = RichHandler(console=Console(stderr=True), show_path=False)
|
|
245
|
+
logging.basicConfig(level=logging.INFO, format='%(message)s', handlers=[handler])
|
|
246
|
+
# huggingface_hub logs an HTTP request on every model load.
|
|
247
|
+
logging.getLogger('httpx').setLevel(logging.WARNING)
|
|
248
|
+
fire.Fire(Scope)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Groups similar texts locally (static embeddings + HDBSCAN) and ranks the groups by size."""
|
|
2
|
+
|
|
3
|
+
import collections
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
_LOGGER = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
# Small (about 30 MB), numpy only, runs on a CPU in seconds; nothing leaves the machine.
|
|
12
|
+
_MODEL = 'minishlab/potion-base-8M'
|
|
13
|
+
_MAX_CHARS = 300
|
|
14
|
+
_NORMALISE = [
|
|
15
|
+
(re.compile(r'https?://\S+'), '<url>'),
|
|
16
|
+
(re.compile(r'(?:[a-z]:)?[\\/][\w.\\/~-]+', re.IGNORECASE), '<path>'),
|
|
17
|
+
(re.compile(r'[!#]\d+'), '<ref>'),
|
|
18
|
+
(re.compile(r'\b[0-9a-f]{7,40}\b'), '<hash>'),
|
|
19
|
+
(re.compile(r'\d+'), '<n>'),
|
|
20
|
+
(re.compile(r'\s+'), ' '),
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class Cluster:
|
|
26
|
+
total: int
|
|
27
|
+
variants: list[tuple[str, int]] # most frequent first
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def normalise(text: str) -> str:
|
|
31
|
+
"""Returns `text` lowercased, with paths, URLs, references, hashes and numbers replaced by placeholders."""
|
|
32
|
+
text = text.lower()[:_MAX_CHARS]
|
|
33
|
+
for pattern, placeholder in _NORMALISE:
|
|
34
|
+
text = pattern.sub(placeholder, text)
|
|
35
|
+
return text.strip()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def rank(texts: Iterable[str], *, min_size: int) -> tuple[list[Cluster], int]:
|
|
39
|
+
"""Returns clusters of at least `min_size` occurrences, largest first, and the count of one-offs."""
|
|
40
|
+
counts = collections.Counter(texts)
|
|
41
|
+
total = sum(counts.values())
|
|
42
|
+
unique = list(counts)
|
|
43
|
+
labels = _labels(unique, counts=counts, min_size=min_size) if total >= min_size and len(unique) > 1 else []
|
|
44
|
+
groups = collections.defaultdict(list)
|
|
45
|
+
for text, label in zip(unique, labels or [-1] * len(unique)):
|
|
46
|
+
if label != -1:
|
|
47
|
+
groups[label].append(text)
|
|
48
|
+
elif counts[text] >= min_size:
|
|
49
|
+
# A frequent exact repeat is a cluster even when nothing else is like it.
|
|
50
|
+
groups[('exact', text)].append(text)
|
|
51
|
+
clusters = [
|
|
52
|
+
Cluster(
|
|
53
|
+
total=sum(counts[t] for t in members),
|
|
54
|
+
variants=sorted(((t, counts[t]) for t in members), key=lambda v: (-v[1], v[0])),
|
|
55
|
+
)
|
|
56
|
+
for members in groups.values()
|
|
57
|
+
]
|
|
58
|
+
clusters.sort(key=lambda c: (-c.total, c.variants[0][0]))
|
|
59
|
+
return clusters, total - sum(c.total for c in clusters)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _labels(unique: list[str], *, counts: collections.Counter, min_size: int) -> list[int]:
|
|
63
|
+
"""Returns an HDBSCAN label per unique text, weighting each text by how often it occurs."""
|
|
64
|
+
import numpy as np
|
|
65
|
+
from model2vec import StaticModel
|
|
66
|
+
from sklearn.cluster import HDBSCAN
|
|
67
|
+
|
|
68
|
+
emb = StaticModel.from_pretrained(_MODEL).encode(unique)
|
|
69
|
+
emb = emb / np.linalg.norm(emb, axis=1, keepdims=True).clip(min=1e-9)
|
|
70
|
+
# HDBSCAN has no sample weights, so repeat each text's row by its count.
|
|
71
|
+
repeats = np.array([counts[t] for t in unique])
|
|
72
|
+
labels = HDBSCAN(min_cluster_size=min_size, min_samples=2, copy=True).fit_predict(np.repeat(emb, repeats, axis=0))
|
|
73
|
+
# Identical rows always share a label; take the first of each run.
|
|
74
|
+
starts = np.concatenate(([0], np.cumsum(repeats)[:-1]))
|
|
75
|
+
_LOGGER.info('Clustered `%s` texts (`%s` unique).', int(repeats.sum()), len(unique))
|
|
76
|
+
return [int(labels[s]) for s in starts]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Remembers findings the user chose to leave alone, so they are not suggested again."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import threading
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import platformdirs
|
|
10
|
+
|
|
11
|
+
from repeat_o_scope.findings import Finding, RepeatOScopeError, Report
|
|
12
|
+
|
|
13
|
+
_LOGGER = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
_LAST_PREFIX = 'last-'
|
|
16
|
+
_DECLINED = 'declined.json'
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def default_dir() -> Path:
|
|
20
|
+
"""Returns the local, never-synced directory for repeat-o-scope's state."""
|
|
21
|
+
return platformdirs.user_data_path('repeat-o-scope', appauthor=False)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def remember_last(*, data_dir: Path, report: Report) -> None:
|
|
25
|
+
"""Replaces the record of `report`'s last run, so `decline` can look its findings up by id."""
|
|
26
|
+
# One file per report: the skill runs the four reports in parallel, and they must not share a file.
|
|
27
|
+
_save(data_dir / f'{_LAST_PREFIX}{report.name}.json', {f.id: [v.text for v in f.variants] for f in report.findings})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def decline(*, data_dir: Path, finding_id: str) -> list[str]:
|
|
31
|
+
"""Marks a finding from the last run of any report as declined and returns its variants."""
|
|
32
|
+
for path in sorted(data_dir.glob(f'{_LAST_PREFIX}*.json')):
|
|
33
|
+
# A cache: a half-written or corrupt one only means nothing is known from that run.
|
|
34
|
+
if (variants := _load(path, lenient=True).get(finding_id)) is not None:
|
|
35
|
+
break
|
|
36
|
+
else:
|
|
37
|
+
raise RepeatOScopeError(f'Unknown finding `{finding_id}`: not in the last run of any report!')
|
|
38
|
+
declined = _load(data_dir / _DECLINED, lenient=False)
|
|
39
|
+
declined[finding_id] = variants
|
|
40
|
+
_save(data_dir / _DECLINED, declined)
|
|
41
|
+
_LOGGER.info('Declined `%s`.', finding_id)
|
|
42
|
+
return variants
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def drop_declined(*, data_dir: Path, findings: list[Finding]) -> list[Finding]:
|
|
46
|
+
"""Returns `findings` without those whose top variant was in a declined finding."""
|
|
47
|
+
# Variants, not ids: as the window slides, a group's top variant and its boundaries shift.
|
|
48
|
+
gone = {text for variants in _load(data_dir / _DECLINED, lenient=False).values() for text in variants}
|
|
49
|
+
return [f for f in findings if f.variants[0].text not in gone]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _load(path: Path, *, lenient: bool) -> dict[str, list[str]]:
|
|
53
|
+
if not path.exists():
|
|
54
|
+
return {}
|
|
55
|
+
try:
|
|
56
|
+
return json.loads(path.read_text(encoding='utf-8'))
|
|
57
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as err:
|
|
58
|
+
if lenient:
|
|
59
|
+
_LOGGER.warning('Ignoring unreadable `%s`.', path)
|
|
60
|
+
return {}
|
|
61
|
+
# The user's decisions: never silently forget them.
|
|
62
|
+
raise RepeatOScopeError(f'Cannot read {path}; fix or delete it: {err}!') from err
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _save(path: Path, data: dict[str, list[str]]) -> None:
|
|
66
|
+
"""Writes `data` atomically, so a concurrent reader never sees a half-written file."""
|
|
67
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
tmp = path.with_name(f'{path.name}.{os.getpid()}.{threading.get_ident()}.tmp')
|
|
69
|
+
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
70
|
+
os.replace(tmp, path)
|