repeat-o-scope 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repeat_o_scope/__init__.py +0 -0
- repeat_o_scope/audit.py +136 -0
- repeat_o_scope/cli.py +248 -0
- repeat_o_scope/cluster.py +76 -0
- repeat_o_scope/declined.py +70 -0
- repeat_o_scope/doctor.py +211 -0
- repeat_o_scope/findings.py +71 -0
- repeat_o_scope/install.py +103 -0
- repeat_o_scope/judge.py +67 -0
- repeat_o_scope/judges.py +218 -0
- repeat_o_scope/redaction.py +16 -0
- repeat_o_scope/render.py +259 -0
- repeat_o_scope/skill/SKILL.md +84 -0
- repeat_o_scope/transcripts.py +190 -0
- repeat_o_scope/waste.py +262 -0
- repeat_o_scope/worklog.py +233 -0
- repeat_o_scope-0.1.0.dist-info/METADATA +255 -0
- repeat_o_scope-0.1.0.dist-info/RECORD +21 -0
- repeat_o_scope-0.1.0.dist-info/WHEEL +4 -0
- repeat_o_scope-0.1.0.dist-info/entry_points.txt +4 -0
- repeat_o_scope-0.1.0.dist-info/licenses/LICENSE +21 -0
repeat_o_scope/doctor.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Checks a Claude Code config against the budget checklist in docs/why.md."""
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from repeat_o_scope.install import load_settings
|
|
11
|
+
|
|
12
|
+
_LOGGER = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
CHECKS = (
|
|
15
|
+
'effort',
|
|
16
|
+
'skill_overrides',
|
|
17
|
+
'skills',
|
|
18
|
+
'status_line',
|
|
19
|
+
'scope_rule',
|
|
20
|
+
'read_once_rule',
|
|
21
|
+
'idle_compact',
|
|
22
|
+
'autocompact_override',
|
|
23
|
+
)
|
|
24
|
+
# The compaction checks follow https://github.com/inevolin/claude-code-usage-analysis.
|
|
25
|
+
_IDLE_COMPACT = 'CLAUDE_CODE_IDLE_COMPACT_MIN_TOKENS'
|
|
26
|
+
_IDLE_COMPACT_MAX = 100_000
|
|
27
|
+
_AUTOCOMPACT_PCT = 'CLAUDE_AUTOCOMPACT_PCT_OVERRIDE'
|
|
28
|
+
# On 200k, 25 fires at 45,000 and thrashes; 50 fires at 100,000, well above what a summary leaves.
|
|
29
|
+
_AUTOCOMPACT_MIN_PCT = 50
|
|
30
|
+
_LOW_EFFORT = {'low', 'medium'}
|
|
31
|
+
# docs/why.md gives no number; above this, skill descriptions are a noticeable fixed cost per turn.
|
|
32
|
+
_MAX_SKILLS = 30
|
|
33
|
+
# The two rules docs/why.md says to copy first, matched by their key phrases; the fix is the rule's text there.
|
|
34
|
+
_RULES = {
|
|
35
|
+
'scope_rule': (
|
|
36
|
+
re.compile(r'whole scope|out of scope|\(OOS\)', re.IGNORECASE),
|
|
37
|
+
'Add to CLAUDE.md: "The issue is the whole scope. Repo rules shape the code but never add work; '
|
|
38
|
+
"anything extra goes in the summary as 'left out (OOS)'.\"",
|
|
39
|
+
),
|
|
40
|
+
'read_once_rule': (
|
|
41
|
+
re.compile(r'read,? edit,? read,? edit|read (?:\w+ ){0,3}once|in one pass', re.IGNORECASE),
|
|
42
|
+
'Add to CLAUDE.md: "No read, edit, read, edit loop. List every file needed, read them in one pass, '
|
|
43
|
+
'make all edits, run the tests once."',
|
|
44
|
+
),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclasses.dataclass(frozen=True)
|
|
49
|
+
class Check:
|
|
50
|
+
name: str
|
|
51
|
+
ok: bool
|
|
52
|
+
found: str
|
|
53
|
+
fix: str
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def diagnose(*, claude_dir: Path, project_dir: Path) -> list[Check]:
|
|
57
|
+
"""Returns one check per `CHECKS` name, failing ones first."""
|
|
58
|
+
layers = _layers(claude_dir=claude_dir, project_dir=project_dir)
|
|
59
|
+
done = {
|
|
60
|
+
'effort': _effort(layers),
|
|
61
|
+
'skill_overrides': _skill_overrides(layers),
|
|
62
|
+
'skills': _skills(claude_dir=claude_dir, layers=layers, project_dir=project_dir),
|
|
63
|
+
'status_line': _status_line(layers),
|
|
64
|
+
'idle_compact': _idle_compact(layers),
|
|
65
|
+
'autocompact_override': _autocompact_override(layers),
|
|
66
|
+
}
|
|
67
|
+
claude_mds = [claude_dir / 'CLAUDE.md', project_dir / 'CLAUDE.md', project_dir / '.claude' / 'CLAUDE.md']
|
|
68
|
+
texts = [(p, p.read_text(encoding='utf-8-sig', errors='replace')) for p in claude_mds if p.exists()]
|
|
69
|
+
for name, (pattern, fix) in _RULES.items():
|
|
70
|
+
where = next((p for p, text in texts if pattern.search(text)), None)
|
|
71
|
+
done[name] = Check(name, True, _shown(where), '') if where else Check(name, False, 'not found', fix)
|
|
72
|
+
checks = [done[name] for name in CHECKS]
|
|
73
|
+
checks.sort(key=lambda c: c.ok)
|
|
74
|
+
_LOGGER.info('Checked %s settings, %s to fix.', len(checks), sum(not c.ok for c in checks))
|
|
75
|
+
return checks
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _layers(*, claude_dir: Path, project_dir: Path) -> list[tuple[Path, dict]]:
|
|
79
|
+
"""Returns the settings files that exist, highest precedence first."""
|
|
80
|
+
paths = [
|
|
81
|
+
project_dir / '.claude' / 'settings.local.json',
|
|
82
|
+
project_dir / '.claude' / 'settings.json',
|
|
83
|
+
claude_dir / 'settings.json',
|
|
84
|
+
]
|
|
85
|
+
return [(p, load_settings(p)) for p in paths if p.exists()]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _effort(layers: list[tuple[Path, dict]]) -> Check:
|
|
89
|
+
for path, settings in layers:
|
|
90
|
+
per_model = settings.get('modelSettings')
|
|
91
|
+
per_model = per_model.values() if isinstance(per_model, dict) else []
|
|
92
|
+
levels = [settings.get('effortLevel')] + [m.get('effortLevel') for m in per_model if isinstance(m, dict)]
|
|
93
|
+
levels = sorted({str(level) for level in levels if level})
|
|
94
|
+
if levels:
|
|
95
|
+
ok = set(levels) <= _LOW_EFFORT
|
|
96
|
+
fix = '' if ok else 'Set effort to medium in /config.'
|
|
97
|
+
return Check('effort', ok, f'{", ".join(levels)} ({_shown(path)})', fix)
|
|
98
|
+
return Check('effort', False, 'not set', 'Set effort to medium in /config.')
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _skill_overrides(layers: list[tuple[Path, dict]]) -> Check:
|
|
102
|
+
off = {name for name, state in _merged(layers, key='skillOverrides').items() if state == 'off'}
|
|
103
|
+
source = next((path for path, settings in layers if settings.get('skillOverrides')), None)
|
|
104
|
+
if not off:
|
|
105
|
+
return Check('skill_overrides', False, 'not set', 'Turn off skills you never use with skillOverrides in settings.json.')
|
|
106
|
+
return Check('skill_overrides', True, f'{len(off)} skills off ({_shown(source)})', '')
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _skills(*, claude_dir: Path, layers: list[tuple[Path, dict]], project_dir: Path) -> Check:
|
|
110
|
+
names = [p.parent.name for d in (claude_dir / 'skills', project_dir / '.claude' / 'skills') for p in d.glob('*/SKILL.md')]
|
|
111
|
+
enabled = {key for key, on in _merged(layers, key='enabledPlugins').items() if on is True}
|
|
112
|
+
for key, install_path in _install_paths(claude_dir).items():
|
|
113
|
+
if key in enabled:
|
|
114
|
+
plugin = key.split('@')[0]
|
|
115
|
+
names += [f'{plugin}:{p.parent.name}' for p in Path(install_path).glob('skills/*/SKILL.md')]
|
|
116
|
+
off = {name for name, state in _merged(layers, key='skillOverrides').items() if state == 'off'}
|
|
117
|
+
# An override names a skill bare (`plan`) or with its plugin (`sp:plan`).
|
|
118
|
+
count = sum(name not in off and name.split(':')[-1] not in off for name in names)
|
|
119
|
+
# Built-in and claude.ai skills are not on disk, so the real number is higher.
|
|
120
|
+
found = f'at least {count} enabled'
|
|
121
|
+
if count > _MAX_SKILLS:
|
|
122
|
+
return Check('skills', False, found, "Turn off skills you never use: every enabled skill's description is in context on every turn.")
|
|
123
|
+
return Check('skills', True, found, '')
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _status_line(layers: list[tuple[Path, dict]]) -> Check:
|
|
127
|
+
for path, settings in layers:
|
|
128
|
+
status_line = settings.get('statusLine')
|
|
129
|
+
if isinstance(status_line, dict) and status_line.get('command'):
|
|
130
|
+
# The file, not the command: a command line may carry a token.
|
|
131
|
+
return Check('status_line', True, _shown(path), '')
|
|
132
|
+
return Check('status_line', False, 'not set', 'Add context % and usage % to your status line.')
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _idle_compact(layers: list[tuple[Path, dict]]) -> Check:
|
|
136
|
+
# Idle compact runs only below the 200,000 default if this is lowered; it needs the 1-hour cache of a subscription.
|
|
137
|
+
fix = f'If you take long breaks with the terminal open, set {_IDLE_COMPACT}={_IDLE_COMPACT_MAX} in settings.json env.'
|
|
138
|
+
if (env := _env(layers, name=_IDLE_COMPACT)) is None:
|
|
139
|
+
return Check('idle_compact', False, 'not set', fix)
|
|
140
|
+
value, source = env
|
|
141
|
+
ok = value is not None and value <= _IDLE_COMPACT_MAX
|
|
142
|
+
return Check('idle_compact', ok, f'{"set" if value is None else value} ({source})', '' if ok else fix)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _autocompact_override(layers: list[tuple[Path, dict]]) -> Check:
|
|
146
|
+
if (env := _env(layers, name=_AUTOCOMPACT_PCT)) is None:
|
|
147
|
+
return Check('autocompact_override', True, 'not set', '')
|
|
148
|
+
value, source = env
|
|
149
|
+
model = next((s['model'] for _, s in layers if isinstance(s.get('model'), str)), '')
|
|
150
|
+
# `[1m]` is how Claude Code names a 1M-context model; anything else is taken as 200k.
|
|
151
|
+
ok = model.lower().endswith('[1m]') or (value is not None and value >= _AUTOCOMPACT_MIN_PCT)
|
|
152
|
+
fix = (
|
|
153
|
+
f'Remove {_AUTOCOMPACT_PCT}: on a 200k-context model it compacts below what a summary leaves, '
|
|
154
|
+
'and the session thrashes.'
|
|
155
|
+
)
|
|
156
|
+
return Check('autocompact_override', ok, f'{"set" if value is None else value} ({source})', '' if ok else fix)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _env(layers: list[tuple[Path, dict]], *, name: str) -> tuple[int | None, str] | None:
|
|
160
|
+
"""Returns a variable's value, `None` unless it is an int, and where it was set; `None` if unset."""
|
|
161
|
+
# Only an int is shown: `env` holds secrets, and a value that isn't a number may be one.
|
|
162
|
+
for path, settings in layers:
|
|
163
|
+
env = settings.get('env')
|
|
164
|
+
if isinstance(env, dict) and name in env:
|
|
165
|
+
return _int(env[name]), _shown(path)
|
|
166
|
+
if name in os.environ:
|
|
167
|
+
return _int(os.environ[name]), 'environment'
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _int(value) -> int | None:
|
|
172
|
+
try:
|
|
173
|
+
return int(value)
|
|
174
|
+
except (TypeError, ValueError):
|
|
175
|
+
return None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _install_paths(claude_dir: Path) -> dict[str, str]:
|
|
179
|
+
"""Returns each installed plugin's `installPath`; none when the file is missing or unreadable."""
|
|
180
|
+
path = claude_dir / 'plugins' / 'installed_plugins.json'
|
|
181
|
+
try:
|
|
182
|
+
plugins = json.loads(path.read_text(encoding='utf-8-sig')).get('plugins')
|
|
183
|
+
except (OSError, ValueError, AttributeError) as err:
|
|
184
|
+
# Plugins are optional: a missing or odd file means no plugin skills, not a broken audit.
|
|
185
|
+
_LOGGER.debug('No plugin skills from `%s`: %s', path, err)
|
|
186
|
+
return {}
|
|
187
|
+
if not isinstance(plugins, dict):
|
|
188
|
+
return {}
|
|
189
|
+
paths = {}
|
|
190
|
+
for key, entries in plugins.items():
|
|
191
|
+
# One odd entry loses only that plugin's skills.
|
|
192
|
+
entry = entries[0] if isinstance(entries, list) and entries else None
|
|
193
|
+
if isinstance(entry, dict) and isinstance(entry.get('installPath'), str):
|
|
194
|
+
paths[key] = entry['installPath']
|
|
195
|
+
return paths
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _merged(layers: list[tuple[Path, dict]], *, key: str) -> dict:
|
|
199
|
+
"""Returns the dict setting `key` merged over all layers, higher precedence winning per entry."""
|
|
200
|
+
merged = {}
|
|
201
|
+
for _, settings in reversed(layers):
|
|
202
|
+
if isinstance(settings.get(key), dict):
|
|
203
|
+
merged |= settings[key]
|
|
204
|
+
return merged
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _shown(path: Path) -> str:
|
|
208
|
+
try:
|
|
209
|
+
return str(Path('~') / path.relative_to(Path.home()))
|
|
210
|
+
except ValueError:
|
|
211
|
+
return str(path)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""What a report finds: ranked findings a person or a skill can act on."""
|
|
2
|
+
|
|
3
|
+
import uuid
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from repeat_o_scope.cluster import Cluster
|
|
8
|
+
from repeat_o_scope.waste import Tally
|
|
9
|
+
|
|
10
|
+
# Fixed so that a finding keeps its id across runs and machines.
|
|
11
|
+
_NAMESPACE = uuid.UUID('6f2c5b1e-8d4a-4c3e-9a7b-2e1f0d9c8b7a')
|
|
12
|
+
_MAX_EXAMPLES = 3
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class RepeatOScopeError(ValueError):
|
|
16
|
+
"""Raised on input repeat-o-scope cannot act on."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Variant:
|
|
21
|
+
text: str
|
|
22
|
+
count: int
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Finding:
|
|
27
|
+
id: str
|
|
28
|
+
# prompt | command | ask | failure | denial, or a `waste.KINDS` kind in an audit
|
|
29
|
+
kind: str
|
|
30
|
+
total: int
|
|
31
|
+
variants: list[Variant]
|
|
32
|
+
examples: list[str]
|
|
33
|
+
label: str | None
|
|
34
|
+
# What the wasted calls behind an audit finding cost, and their share of all usage.
|
|
35
|
+
cost: Tally | None = None
|
|
36
|
+
share: float | None = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Report:
|
|
41
|
+
name: str
|
|
42
|
+
days: int
|
|
43
|
+
jev: bool
|
|
44
|
+
totals: dict[str, int]
|
|
45
|
+
findings: list[Finding]
|
|
46
|
+
note: str = field(default='')
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def finding_id(*, report: str, top_variant: str) -> str:
|
|
50
|
+
"""Returns a stable id for the finding whose most frequent variant is `top_variant`."""
|
|
51
|
+
return str(uuid.uuid5(_NAMESPACE, f'{report}:{top_variant}'))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def from_cluster(
|
|
55
|
+
*,
|
|
56
|
+
cluster: Cluster,
|
|
57
|
+
kind: str,
|
|
58
|
+
label: str | None = None,
|
|
59
|
+
originals: Mapping[str, list[str]],
|
|
60
|
+
report: str,
|
|
61
|
+
) -> Finding:
|
|
62
|
+
"""Returns the finding for `cluster`, with examples from the originals behind its top variant."""
|
|
63
|
+
top = cluster.variants[0][0]
|
|
64
|
+
return Finding(
|
|
65
|
+
id=finding_id(report=report, top_variant=top),
|
|
66
|
+
kind=kind,
|
|
67
|
+
total=cluster.total,
|
|
68
|
+
variants=[Variant(text, count) for text, count in cluster.variants],
|
|
69
|
+
examples=list(dict.fromkeys(originals.get(top, [])))[:_MAX_EXAMPLES],
|
|
70
|
+
label=label,
|
|
71
|
+
)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Installs the repeat-o-scope skill and the worklog Stop hook into a Claude Code config directory."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
import shutil
|
|
7
|
+
from importlib import resources
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from repeat_o_scope.findings import RepeatOScopeError
|
|
11
|
+
|
|
12
|
+
_LOGGER = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
_HOOK = {'type': 'command', 'command': 'repeat-o-scope-worklog'}
|
|
15
|
+
# The PowerShell hook this package replaces: two loggers would log every stretch twice.
|
|
16
|
+
# A whole file name only, so a user's own `my-team-worklog.ps1` survives.
|
|
17
|
+
_REPLACED = re.compile(r'(^|[\\/\s"\'])worklog\.ps1\b')
|
|
18
|
+
_BACKUP_SUFFIX = '.repeat-o-scope.bak'
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def install(*, claude_dir: Path) -> list[str]:
|
|
22
|
+
"""Copies the skill and registers the Stop hook; returns what changed."""
|
|
23
|
+
settings_path = claude_dir / 'settings.json'
|
|
24
|
+
settings = load_settings(settings_path)
|
|
25
|
+
stop = settings.setdefault('hooks', {}).setdefault('Stop', [])
|
|
26
|
+
changes = []
|
|
27
|
+
for group in stop:
|
|
28
|
+
kept = [h for h in group.get('hooks', []) if not _REPLACED.search(h.get('command', ''))]
|
|
29
|
+
if len(kept) != len(group.get('hooks', [])):
|
|
30
|
+
changes.append(f'Removed the `worklog.ps1` Stop hook from {settings_path}.')
|
|
31
|
+
group['hooks'] = kept
|
|
32
|
+
stop[:] = [g for g in stop if g.get('hooks')]
|
|
33
|
+
if not any(_is_ours(h) for g in stop for h in g['hooks']):
|
|
34
|
+
stop.append({'hooks': [dict(_HOOK)]})
|
|
35
|
+
changes.append(f'Added the `{_HOOK["command"]}` Stop hook to {settings_path}.')
|
|
36
|
+
if changes:
|
|
37
|
+
_save(settings_path, settings)
|
|
38
|
+
|
|
39
|
+
skill = _skill_path(claude_dir)
|
|
40
|
+
source = resources.files('repeat_o_scope').joinpath('skill', 'SKILL.md').read_text(encoding='utf-8')
|
|
41
|
+
if not skill.exists() or skill.read_text(encoding='utf-8') != source:
|
|
42
|
+
skill.parent.mkdir(parents=True, exist_ok=True)
|
|
43
|
+
skill.write_text(source, encoding='utf-8')
|
|
44
|
+
changes.append(f'Wrote the skill to {skill}.')
|
|
45
|
+
for change in changes:
|
|
46
|
+
_LOGGER.info('%s', change)
|
|
47
|
+
return changes
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def uninstall(*, claude_dir: Path) -> list[str]:
|
|
51
|
+
"""Removes the Stop hook and the skill; returns what changed."""
|
|
52
|
+
settings_path = claude_dir / 'settings.json'
|
|
53
|
+
settings = load_settings(settings_path)
|
|
54
|
+
changes = []
|
|
55
|
+
hooks = settings.get('hooks', {})
|
|
56
|
+
stop = hooks.get('Stop', [])
|
|
57
|
+
for group in stop:
|
|
58
|
+
if any(_is_ours(h) for h in group.get('hooks', [])):
|
|
59
|
+
group['hooks'] = [h for h in group['hooks'] if not _is_ours(h)]
|
|
60
|
+
changes.append(f'Removed the `{_HOOK["command"]}` Stop hook from {settings_path}.')
|
|
61
|
+
stop[:] = [g for g in stop if g.get('hooks')]
|
|
62
|
+
if 'Stop' in hooks and not stop:
|
|
63
|
+
del hooks['Stop']
|
|
64
|
+
if 'hooks' in settings and not hooks:
|
|
65
|
+
del settings['hooks']
|
|
66
|
+
if changes:
|
|
67
|
+
_save(settings_path, settings)
|
|
68
|
+
skill = _skill_path(claude_dir)
|
|
69
|
+
if skill.parent.exists():
|
|
70
|
+
shutil.rmtree(skill.parent)
|
|
71
|
+
changes.append(f'Removed the skill from {skill.parent}.')
|
|
72
|
+
return changes
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _is_ours(hook: dict) -> bool:
|
|
76
|
+
# By command, not the whole entry: a user may have added `timeout` or other keys to it.
|
|
77
|
+
return hook.get('command') == _HOOK['command']
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _skill_path(claude_dir: Path) -> Path:
|
|
81
|
+
return claude_dir / 'skills' / 'repeat-o-scope' / 'SKILL.md'
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def load_settings(path: Path) -> dict:
|
|
85
|
+
"""Returns a settings file as a dict, or an empty one if it does not exist."""
|
|
86
|
+
if not path.exists():
|
|
87
|
+
return {}
|
|
88
|
+
try:
|
|
89
|
+
settings = json.loads(path.read_text(encoding='utf-8-sig'))
|
|
90
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as err:
|
|
91
|
+
# Never rewrite a file we cannot parse: the user's other settings would be lost.
|
|
92
|
+
raise RepeatOScopeError(f'Cannot parse {path}, leaving it untouched: {err}!') from err
|
|
93
|
+
if not isinstance(settings, dict):
|
|
94
|
+
raise RepeatOScopeError(f'Unexpected `{type(settings).__name__}` at the top of {path}!')
|
|
95
|
+
return settings
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _save(path: Path, settings: dict) -> None:
|
|
99
|
+
backup = path.with_name(path.name + _BACKUP_SUFFIX)
|
|
100
|
+
if path.exists() and not backup.exists():
|
|
101
|
+
shutil.copy2(path, backup)
|
|
102
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
103
|
+
path.write_text(json.dumps(settings, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
repeat_o_scope/judge.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Typed judgments where clustering alone cannot tell things apart."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
from typesafe_sdk import Choice, Noul
|
|
7
|
+
|
|
8
|
+
from repeat_o_scope.judges import Judge
|
|
9
|
+
|
|
10
|
+
_LOGGER = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
_CONCURRENCY = 8
|
|
13
|
+
_ASK = Choice(
|
|
14
|
+
instructions=(
|
|
15
|
+
'This is the last line of an AI coding assistant\'s turn, and the user answered it only with a go-ahead '
|
|
16
|
+
'such as "ship", "go" or "yes". What kind of message is `closing`?'
|
|
17
|
+
),
|
|
18
|
+
criteria={
|
|
19
|
+
'permission': (
|
|
20
|
+
'Asks for permission or a go-ahead for a next step the assistant could have taken by itself: push, '
|
|
21
|
+
'commit, ship, post, run, continue. Includes "say "ship" and I\'ll …", "reply go and …", "want me to …?".'
|
|
22
|
+
),
|
|
23
|
+
'choice': 'Asks the user to pick between real alternatives that change what gets built or done.',
|
|
24
|
+
'clarifying': 'Asks for a fact or a preference that only the user knows.',
|
|
25
|
+
'review': 'Asks the user to look at and approve something shown: a design, a draft, a diff, a plan.',
|
|
26
|
+
'report': 'Asks for nothing: states a result, a status, a caveat or a list item.',
|
|
27
|
+
},
|
|
28
|
+
)
|
|
29
|
+
_REAL = Noul(
|
|
30
|
+
instructions=(
|
|
31
|
+
'An AI coding assistant made a tool call, and the result was flagged as an error. `failure` reads '
|
|
32
|
+
'"<tool>: <what it tried> → <first line of the error>". Did the attempt really fail to do what it was '
|
|
33
|
+
'meant to do? Answer no when the error is only normal output written to stderr, a warning, a progress '
|
|
34
|
+
'line, or a non-zero exit the command was expected to produce, such as a test run or a linter reporting '
|
|
35
|
+
'what it found.'
|
|
36
|
+
),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def kinds_of_asks(closings: list[str], *, judge: Judge) -> list[str]:
|
|
41
|
+
"""Returns the kind of ask (a key of `_ASK.criteria`) for each closing line."""
|
|
42
|
+
answers = _ask_all([({'closing': c}, {'kind': _ASK}) for c in closings], judge=judge)
|
|
43
|
+
return [a['kind'].choice for a in answers]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def real_failures(failures: list[str], *, judge: Judge) -> list[bool]:
|
|
47
|
+
"""Returns, for each failure ("tool: attempt → error"), whether it really failed rather than wrote to stderr."""
|
|
48
|
+
answers = _ask_all([({'failure': f}, {'real': _REAL}) for f in failures], judge=judge)
|
|
49
|
+
return [a['real'].noul >= 0.5 for a in answers]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _ask_all(requests: list[tuple[dict, dict]], *, judge: Judge) -> list[dict]:
|
|
53
|
+
"""Returns the answers to independent requests, run concurrently in request order."""
|
|
54
|
+
|
|
55
|
+
async def run():
|
|
56
|
+
limit = asyncio.Semaphore(_CONCURRENCY)
|
|
57
|
+
async with judge:
|
|
58
|
+
|
|
59
|
+
async def one(state, questions):
|
|
60
|
+
async with limit:
|
|
61
|
+
return await judge.ask(state=state, questions=questions)
|
|
62
|
+
|
|
63
|
+
return await asyncio.gather(*(one(s, q) for s, q in requests))
|
|
64
|
+
|
|
65
|
+
answers = asyncio.run(run())
|
|
66
|
+
_LOGGER.info('Asked `%s` `%s` questions.', judge.name, len(answers))
|
|
67
|
+
return answers
|
repeat_o_scope/judges.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Opens the first judge that answers: Jev, then Gemini Decision 1, then Clef."""
|
|
2
|
+
|
|
3
|
+
import abc
|
|
4
|
+
import asyncio
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import override
|
|
10
|
+
|
|
11
|
+
from repeat_o_scope.findings import RepeatOScopeError
|
|
12
|
+
|
|
13
|
+
_LOGGER = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
_TIMEOUT_SECONDS = 30
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Answer:
|
|
20
|
+
choice: str | None
|
|
21
|
+
noul: float | None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Judge(abc.ABC):
|
|
25
|
+
"""Answers typed questions (TypeSafe `Choice` and `Noul`) about a state; use inside `async with`."""
|
|
26
|
+
|
|
27
|
+
name: str = NotImplemented
|
|
28
|
+
env: tuple[str, ...] = NotImplemented
|
|
29
|
+
|
|
30
|
+
def available(self) -> bool:
|
|
31
|
+
return all(os.environ.get(name) for name in self.env)
|
|
32
|
+
|
|
33
|
+
async def __aenter__(self):
|
|
34
|
+
return self
|
|
35
|
+
|
|
36
|
+
async def __aexit__(self, *exc):
|
|
37
|
+
return False
|
|
38
|
+
|
|
39
|
+
async def ask(self, *, questions: dict, state: dict) -> dict[str, Answer]:
|
|
40
|
+
"""Returns one answer per question, or raises naming this judge."""
|
|
41
|
+
try:
|
|
42
|
+
answers = await self._ask(questions=questions, state=state)
|
|
43
|
+
except Exception as err:
|
|
44
|
+
raise RepeatOScopeError(f'Judge `{self.name}` failed: {err}') from err
|
|
45
|
+
if set(answers) != set(questions):
|
|
46
|
+
raise RepeatOScopeError(f'Judge `{self.name}` answered `{sorted(answers)}`, asked `{sorted(questions)}`!')
|
|
47
|
+
for key, question in questions.items():
|
|
48
|
+
if not _answers(question, answers[key]):
|
|
49
|
+
raise RepeatOScopeError(f'Judge `{self.name}` gave no usable answer to `{key}`: `{answers[key]}`!')
|
|
50
|
+
return answers
|
|
51
|
+
|
|
52
|
+
@abc.abstractmethod
|
|
53
|
+
async def _ask(self, *, questions: dict, state: dict) -> dict[str, Answer]: ...
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _Jev(Judge):
|
|
57
|
+
name = 'jev'
|
|
58
|
+
# vainu-data names the key TYPESAFE_AI_API_KEY.
|
|
59
|
+
_KEYS = ('TYPESAFE_API_KEY', 'TYPESAFE_AI_API_KEY')
|
|
60
|
+
|
|
61
|
+
@override
|
|
62
|
+
def available(self) -> bool:
|
|
63
|
+
return any(os.environ.get(k) for k in self._KEYS)
|
|
64
|
+
|
|
65
|
+
@override
|
|
66
|
+
async def __aenter__(self):
|
|
67
|
+
from typesafe_sdk import AsyncTypeSafeClient
|
|
68
|
+
|
|
69
|
+
api_key = next(os.environ[k] for k in self._KEYS if os.environ.get(k))
|
|
70
|
+
self._client = AsyncTypeSafeClient(api_key=api_key, timeout=_TIMEOUT_SECONDS)
|
|
71
|
+
await self._client.__aenter__()
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
@override
|
|
75
|
+
async def __aexit__(self, *exc):
|
|
76
|
+
return await self._client.__aexit__(*exc)
|
|
77
|
+
|
|
78
|
+
@override
|
|
79
|
+
async def _ask(self, *, questions: dict, state: dict) -> dict[str, Answer]:
|
|
80
|
+
r = await self._client.system_one(state=state, questions=questions)
|
|
81
|
+
return {k: Answer(choice=c.choice, noul=None) for k, c in r.choices.items()} | {
|
|
82
|
+
k: Answer(choice=None, noul=n.noul) for k, n in r.nouls.items()
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class _Wire(Judge):
|
|
87
|
+
"""A judge behind plain HTTP that takes Jev's question format."""
|
|
88
|
+
|
|
89
|
+
@override
|
|
90
|
+
async def __aenter__(self):
|
|
91
|
+
self._client = http_client(headers=self._headers())
|
|
92
|
+
return self
|
|
93
|
+
|
|
94
|
+
@override
|
|
95
|
+
async def __aexit__(self, *exc):
|
|
96
|
+
await self._client.aclose()
|
|
97
|
+
return False
|
|
98
|
+
|
|
99
|
+
@override
|
|
100
|
+
async def _ask(self, *, questions: dict, state: dict) -> dict[str, Answer]:
|
|
101
|
+
wire = {k: _to_wire(q) for k, q in questions.items()}
|
|
102
|
+
response = await self._client.post(self._url(), json=self._body(questions=wire, state=state))
|
|
103
|
+
response.raise_for_status()
|
|
104
|
+
return {
|
|
105
|
+
k: Answer(choice=a.get('choice'), noul=a.get('noul'))
|
|
106
|
+
for k, a in self._answers(response.json()).items()
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
@abc.abstractmethod
|
|
110
|
+
def _headers(self) -> dict: ...
|
|
111
|
+
|
|
112
|
+
@abc.abstractmethod
|
|
113
|
+
def _url(self) -> str: ...
|
|
114
|
+
|
|
115
|
+
@abc.abstractmethod
|
|
116
|
+
def _body(self, *, questions: dict, state: dict) -> dict: ...
|
|
117
|
+
|
|
118
|
+
@abc.abstractmethod
|
|
119
|
+
def _answers(self, doc: dict) -> dict: ...
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class _Gemini(_Wire):
|
|
123
|
+
name = 'gemini-decision-1'
|
|
124
|
+
env = ('GEMINI_API_KEY',)
|
|
125
|
+
|
|
126
|
+
@override
|
|
127
|
+
def _headers(self) -> dict:
|
|
128
|
+
return {'x-goog-api-key': os.environ['GEMINI_API_KEY']}
|
|
129
|
+
|
|
130
|
+
@override
|
|
131
|
+
def _url(self) -> str:
|
|
132
|
+
return 'https://generativelanguage.googleapis.com/v1beta/decisions'
|
|
133
|
+
|
|
134
|
+
@override
|
|
135
|
+
def _body(self, *, questions: dict, state: dict) -> dict:
|
|
136
|
+
# The Decisions API takes one input string, not a state object.
|
|
137
|
+
return {'model': self.name, 'input': json.dumps(state, ensure_ascii=False), 'questions': questions}
|
|
138
|
+
|
|
139
|
+
@override
|
|
140
|
+
def _answers(self, doc: dict) -> dict:
|
|
141
|
+
return doc['answers']
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class _Clef(_Wire):
|
|
145
|
+
name = 'clef'
|
|
146
|
+
env = ('CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_KEY')
|
|
147
|
+
|
|
148
|
+
@override
|
|
149
|
+
def _headers(self) -> dict:
|
|
150
|
+
return {'Authorization': f'Bearer {os.environ["CLOUDFLARE_API_KEY"]}'}
|
|
151
|
+
|
|
152
|
+
@override
|
|
153
|
+
def _url(self) -> str:
|
|
154
|
+
account = os.environ['CLOUDFLARE_ACCOUNT_ID']
|
|
155
|
+
return f'https://api.cloudflare.com/client/v4/accounts/{account}/ai/run/@cf/cloudflare/clef'
|
|
156
|
+
|
|
157
|
+
@override
|
|
158
|
+
def _body(self, *, questions: dict, state: dict) -> dict:
|
|
159
|
+
return {'model': self.name, 'state': state, 'questions': questions}
|
|
160
|
+
|
|
161
|
+
@override
|
|
162
|
+
def _answers(self, doc: dict) -> dict:
|
|
163
|
+
return doc['result']['answers']
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
_JUDGES = (_Jev, _Gemini, _Clef)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def http_client(**kwargs):
|
|
170
|
+
"""Returns the HTTP client the wire judges use."""
|
|
171
|
+
# httpx2 because typesafe-sdk already brings it.
|
|
172
|
+
import httpx2
|
|
173
|
+
|
|
174
|
+
return httpx2.AsyncClient(timeout=_TIMEOUT_SECONDS, **kwargs)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def open_judge() -> Judge | None:
|
|
178
|
+
"""Returns the first judge whose keys are set and that answers a test question; None if none does."""
|
|
179
|
+
for judge_class in _JUDGES:
|
|
180
|
+
judge = judge_class()
|
|
181
|
+
if not judge.available():
|
|
182
|
+
continue
|
|
183
|
+
try:
|
|
184
|
+
asyncio.run(_probe(judge))
|
|
185
|
+
# Any failure, a missing library, a bad key or no network, only means trying the next judge.
|
|
186
|
+
except Exception as err:
|
|
187
|
+
_LOGGER.warning('Judge `%s` unavailable: %s', judge.name, err)
|
|
188
|
+
continue
|
|
189
|
+
_LOGGER.info('Judging with `%s`.', judge.name)
|
|
190
|
+
return judge
|
|
191
|
+
_LOGGER.info('No judge available.')
|
|
192
|
+
return None
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
async def _probe(judge: Judge) -> None:
|
|
196
|
+
from typesafe_sdk import Noul
|
|
197
|
+
|
|
198
|
+
async with judge:
|
|
199
|
+
await judge.ask(state={'a': 2, 'b': 1}, questions={'probe': Noul(instructions='Is `a` greater than `b`?')})
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _answers(question, answer: Answer) -> bool:
|
|
203
|
+
"""Returns whether `answer` is a probability for a noul, or one of the criteria for a choice."""
|
|
204
|
+
from typesafe_sdk import Noul
|
|
205
|
+
|
|
206
|
+
if isinstance(question, Noul):
|
|
207
|
+
return isinstance(answer.noul, (int, float))
|
|
208
|
+
return answer.choice in question.criteria
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _to_wire(question) -> dict:
|
|
212
|
+
from typesafe_sdk import Noul
|
|
213
|
+
|
|
214
|
+
if isinstance(question, Noul):
|
|
215
|
+
# Our nouls carry no criteria; Gemini and Clef would want `true_description` / `false_description`.
|
|
216
|
+
assert question.criteria is None, 'Noul criteria are not sent to wire judges'
|
|
217
|
+
return {'type': 'noul', 'instructions': question.instructions}
|
|
218
|
+
return {'type': 'choice', 'instructions': question.instructions, 'criteria': dict(question.criteria)}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""The one place that decides what counts as a credential."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
# Google OAuth tokens and codes, GitLab, GitHub, Slack, AWS, Anthropic and OpenAI keys, private keys.
|
|
6
|
+
# Case-insensitive because the original worklog.ps1 used PowerShell's -replace, which is.
|
|
7
|
+
_SECRET = re.compile(
|
|
8
|
+
r'ya29\.[\w-]+|\b4/0A[\w-]+|1//0[\w-]{20,}|glpat-[\w.-]+|gh[pousr]_\w{20,}|xox[abprs]-[\w-]+'
|
|
9
|
+
r'|AKIA[0-9A-Z]{16}|sk-(ant-)?[\w-]{20,}|-----BEGIN [A-Z ]*PRIVATE KEY-----',
|
|
10
|
+
re.IGNORECASE,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def redact(text: str) -> str:
|
|
15
|
+
"""Returns `text` with credentials replaced by `[REDACTED]`."""
|
|
16
|
+
return _SECRET.sub('[REDACTED]', text)
|