rulewright 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rulewright/__init__.py +1 -0
- rulewright/cli.py +490 -0
- rulewright/cluster/__init__.py +0 -0
- rulewright/cluster/embed.py +66 -0
- rulewright/cluster/group.py +93 -0
- rulewright/config.py +19 -0
- rulewright/guard/__init__.py +0 -0
- rulewright/guard/emit.py +75 -0
- rulewright/guard/profile.py +172 -0
- rulewright/ingest/__init__.py +0 -0
- rulewright/ingest/git_history.py +185 -0
- rulewright/ingest/reviews.py +1 -0
- rulewright/ingest/sessions/__init__.py +0 -0
- rulewright/ingest/sessions/base.py +47 -0
- rulewright/ingest/sessions/claude_code.py +138 -0
- rulewright/ingest/sessions/codex.py +112 -0
- rulewright/mine/__init__.py +0 -0
- rulewright/mine/detectors.py +256 -0
- rulewright/mine/score.py +28 -0
- rulewright/mine/signals.py +20 -0
- rulewright/pipeline.py +146 -0
- rulewright/pr.py +123 -0
- rulewright/propose/__init__.py +0 -0
- rulewright/propose/guards.py +131 -0
- rulewright/propose/llm.py +141 -0
- rulewright/propose/redact.py +44 -0
- rulewright/propose/writer.py +240 -0
- rulewright/prove/__init__.py +0 -0
- rulewright/prove/agent.py +131 -0
- rulewright/prove/judge.py +62 -0
- rulewright/prove/runner.py +198 -0
- rulewright/prove/sandbox.py +138 -0
- rulewright/prove/tasks.py +85 -0
- rulewright/report.py +63 -0
- rulewright/rules/__init__.py +0 -0
- rulewright/rules/emit/__init__.py +55 -0
- rulewright/rules/emit/agents_md.py +9 -0
- rulewright/rules/emit/base.py +74 -0
- rulewright/rules/emit/claude_md.py +9 -0
- rulewright/rules/emit/copilot.py +9 -0
- rulewright/rules/emit/cursor.py +12 -0
- rulewright/rules/model.py +24 -0
- rulewright/rules/source.py +127 -0
- rulewright/rules/stale.py +142 -0
- rulewright/store.py +56 -0
- rulewright-0.1.0.dist-info/METADATA +144 -0
- rulewright-0.1.0.dist-info/RECORD +50 -0
- rulewright-0.1.0.dist-info/WHEEL +4 -0
- rulewright-0.1.0.dist-info/entry_points.txt +2 -0
- rulewright-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Codex rollout reader (commands only).
|
|
2
|
+
|
|
3
|
+
Rollouts live in ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl. Each line is
|
|
4
|
+
`{"timestamp", "type", "payload"}`. A `session_meta` record carries the working directory
|
|
5
|
+
(`payload.cwd`); `response_item` records with `payload.type == "function_call"` carry the command
|
|
6
|
+
as a JSON string in `payload.arguments`, and `function_call_output` answers it by `call_id`.
|
|
7
|
+
The command is a string, or an argv list such as ["bash", "-lc", "git status"].
|
|
8
|
+
Checked against a real rollout (Codex 0.137); unknown or malformed lines are skipped.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from rulewright.ingest.sessions.base import ToolCall
|
|
18
|
+
|
|
19
|
+
SHELL_TOOLS = {"shell", "shell_command", "exec_command", "local_shell"}
|
|
20
|
+
DENIED_MARKERS = ("rejected by user", "denied by user", "user declined", "aborted by user")
|
|
21
|
+
_EXIT_CODE = re.compile(r"exit code:?\s*(-?\d+)", re.I)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _norm(path: str) -> str:
|
|
25
|
+
return path.replace("\\", "/").rstrip("/").lower()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def command_text(command: Any) -> str | None:
|
|
29
|
+
if isinstance(command, str):
|
|
30
|
+
return command
|
|
31
|
+
if isinstance(command, list) and command and all(isinstance(p, str) for p in command):
|
|
32
|
+
parts: list[str] = command
|
|
33
|
+
if len(parts) >= 3 and parts[1] in ("-c", "-lc", "/c", "-Command"):
|
|
34
|
+
return parts[-1] # the script the shell was asked to run
|
|
35
|
+
return " ".join(parts)
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _outcome(output: Any) -> tuple[bool, bool]:
|
|
40
|
+
"""(failed, denied) from a function_call_output payload."""
|
|
41
|
+
text = output if isinstance(output, str) else json.dumps(output)
|
|
42
|
+
low = text.lower()
|
|
43
|
+
if any(m in low for m in DENIED_MARKERS):
|
|
44
|
+
return False, True
|
|
45
|
+
if low.startswith("execution error"):
|
|
46
|
+
return True, False
|
|
47
|
+
m = _EXIT_CODE.search(text) or re.search(r'"exit_code":\s*(-?\d+)', text)
|
|
48
|
+
return (m is not None and int(m.group(1)) != 0), False
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class CodexSource:
|
|
52
|
+
name = "codex"
|
|
53
|
+
|
|
54
|
+
def __init__(self, sessions_dir: Path | None = None) -> None:
|
|
55
|
+
self.sessions_dir = sessions_dir or Path.home() / ".codex" / "sessions"
|
|
56
|
+
|
|
57
|
+
def read_calls(self, repo: Path) -> list[ToolCall]:
|
|
58
|
+
if not self.sessions_dir.is_dir():
|
|
59
|
+
return []
|
|
60
|
+
want = _norm(str(repo.resolve()))
|
|
61
|
+
calls: list[ToolCall] = []
|
|
62
|
+
for path in sorted(self.sessions_dir.glob("**/rollout-*.jsonl")):
|
|
63
|
+
calls.extend(self._calls_in(path, want))
|
|
64
|
+
calls.sort(key=lambda c: c.ts)
|
|
65
|
+
return calls
|
|
66
|
+
|
|
67
|
+
@staticmethod
|
|
68
|
+
def _calls_in(path: Path, want_cwd: str) -> list[ToolCall]:
|
|
69
|
+
session = path.stem
|
|
70
|
+
pending: dict[str, ToolCall] = {}
|
|
71
|
+
inside = False
|
|
72
|
+
with path.open(encoding="utf-8", errors="replace") as fh:
|
|
73
|
+
for line in fh:
|
|
74
|
+
try:
|
|
75
|
+
rec = json.loads(line)
|
|
76
|
+
payload = rec["payload"]
|
|
77
|
+
kind = rec["type"]
|
|
78
|
+
except (ValueError, KeyError, TypeError):
|
|
79
|
+
continue
|
|
80
|
+
if not isinstance(payload, dict):
|
|
81
|
+
continue
|
|
82
|
+
if kind == "session_meta":
|
|
83
|
+
inside = _norm(str(payload.get("cwd", ""))) == want_cwd
|
|
84
|
+
session = str(payload.get("id") or session)
|
|
85
|
+
continue
|
|
86
|
+
if not inside or kind != "response_item":
|
|
87
|
+
continue
|
|
88
|
+
call = CodexSource._parse(payload, rec.get("timestamp"), session)
|
|
89
|
+
if call is not None:
|
|
90
|
+
pending[str(payload["call_id"])] = call
|
|
91
|
+
elif payload.get("type") == "function_call_output":
|
|
92
|
+
done = pending.get(str(payload.get("call_id")))
|
|
93
|
+
if done is not None:
|
|
94
|
+
done.failed, done.denied = _outcome(payload.get("output"))
|
|
95
|
+
return list(pending.values())
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def _parse(payload: dict[str, Any], stamp: Any, session: str) -> ToolCall | None:
|
|
99
|
+
if payload.get("type") != "function_call" or payload.get("name") not in SHELL_TOOLS:
|
|
100
|
+
return None
|
|
101
|
+
try:
|
|
102
|
+
args = json.loads(payload["arguments"])
|
|
103
|
+
command = command_text(args.get("command") or args.get("cmd"))
|
|
104
|
+
when = datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
|
|
105
|
+
call_id = payload["call_id"]
|
|
106
|
+
except (ValueError, KeyError, TypeError, AttributeError):
|
|
107
|
+
return None
|
|
108
|
+
if command is None or not call_id:
|
|
109
|
+
return None
|
|
110
|
+
return ToolCall(
|
|
111
|
+
session_id=session, agent="codex", tool=str(payload["name"]), command=command, ts=when
|
|
112
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""Detectors turn commit events into Signals. Each has a known right answer in the fixtures.
|
|
2
|
+
|
|
3
|
+
Revert and re-fix matching is by line content: a later commit counts only if it removes lines
|
|
4
|
+
the agent commit added. Counting deletions anywhere in a touched file flagged most commits to
|
|
5
|
+
busy files in real repositories.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from datetime import timedelta
|
|
11
|
+
|
|
12
|
+
from rulewright.ingest.git_history import Commit, FileChange
|
|
13
|
+
from rulewright.ingest.sessions.base import SessionMessage
|
|
14
|
+
from rulewright.mine.signals import Signal
|
|
15
|
+
|
|
16
|
+
REVERT_WINDOW = timedelta(days=7)
|
|
17
|
+
REFIX_WINDOW = timedelta(hours=48)
|
|
18
|
+
EXCERPT_CHARS = 400
|
|
19
|
+
MAX_EXCERPT_LINES = 3
|
|
20
|
+
# A later commit "deletes most of" an agent commit when it removes this share of its added lines.
|
|
21
|
+
MOSTLY_DELETED = 0.5
|
|
22
|
+
_REVERTS_SHA = re.compile(r"This reverts commit ([0-9a-f]{7,40})", re.I)
|
|
23
|
+
# Lockfiles, bundles and the like: edited by tools and everyone, never evidence of a mistake.
|
|
24
|
+
_GENERATED = re.compile(
|
|
25
|
+
r"(^|/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|Cargo\.lock|uv\.lock|poetry\.lock"
|
|
26
|
+
r"|Gemfile\.lock|composer\.lock|go\.sum)$|\.lock$|\.min\.(js|css)$|\.snap$|\.svg$|\.map$",
|
|
27
|
+
re.I,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _significant(line: str) -> bool:
|
|
32
|
+
"""Skip braces, blank lines and other lines too common to identify anything."""
|
|
33
|
+
s = line.strip()
|
|
34
|
+
return len(s) >= 4 and sum(c.isalnum() for c in s) >= 3
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _Added:
|
|
38
|
+
"""The significant lines an agent commit added, per file, indexed once.
|
|
39
|
+
|
|
40
|
+
Pair checks run for every (later commit, agent commit) inside the time window, so building
|
|
41
|
+
this per pair made scanning 500 commits of a busy repository take minutes.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, commit: Commit) -> None:
|
|
45
|
+
self.pools: dict[str, Counter[str]] = {}
|
|
46
|
+
self.total = 0
|
|
47
|
+
for f in commit.files:
|
|
48
|
+
if _GENERATED.search(f.path):
|
|
49
|
+
continue
|
|
50
|
+
pool = Counter(line for line in f.added_lines if _significant(line))
|
|
51
|
+
if pool:
|
|
52
|
+
self.pools[f.path] = pool
|
|
53
|
+
self.total += sum(pool.values())
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _removed_agent_lines(later: Commit, added: _Added) -> dict[str, list[str]]:
|
|
57
|
+
"""Per file, the lines `later` removes that the agent commit added (each matches once)."""
|
|
58
|
+
out: dict[str, list[str]] = {}
|
|
59
|
+
for f in later.files:
|
|
60
|
+
pool = added.pools.get(f.path)
|
|
61
|
+
if pool is None:
|
|
62
|
+
continue
|
|
63
|
+
candidates = [line for line in f.removed_lines if line in pool]
|
|
64
|
+
if not candidates:
|
|
65
|
+
continue
|
|
66
|
+
left = pool.copy()
|
|
67
|
+
hits = []
|
|
68
|
+
for line in candidates:
|
|
69
|
+
if left[line] > 0:
|
|
70
|
+
left[line] -= 1
|
|
71
|
+
hits.append(line)
|
|
72
|
+
if hits:
|
|
73
|
+
out[f.path] = hits
|
|
74
|
+
return out
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _fix_excerpt(later: Commit, removed: dict[str, list[str]]) -> str:
|
|
78
|
+
"""What the human changed: the agent's lines that were removed, and what replaced them."""
|
|
79
|
+
lines: list[str] = []
|
|
80
|
+
for path, hits in list(removed.items())[:2]:
|
|
81
|
+
lines += [f"- {line}" for line in hits[:MAX_EXCERPT_LINES]]
|
|
82
|
+
file = next(f for f in later.files if f.path == path)
|
|
83
|
+
lines += [f"+ {line}" for line in file.added_lines[:MAX_EXCERPT_LINES]]
|
|
84
|
+
return "\n".join(lines)[:EXCERPT_CHARS]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _added_excerpt(target: Commit) -> str:
|
|
88
|
+
lines = [
|
|
89
|
+
f"+ {line}"
|
|
90
|
+
for f in target.files
|
|
91
|
+
if not _GENERATED.search(f.path)
|
|
92
|
+
for line in f.added_lines
|
|
93
|
+
if _significant(line)
|
|
94
|
+
]
|
|
95
|
+
return "\n".join(lines[:MAX_EXCERPT_LINES])[:EXCERPT_CHARS]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def detect_reverts(commits: list[Commit]) -> list[Signal]:
|
|
99
|
+
"""Commits that revert, or delete most of, an agent commit within 7 days."""
|
|
100
|
+
agent = [c for c in commits if c.is_agent]
|
|
101
|
+
added = {c.sha: _Added(c) for c in agent}
|
|
102
|
+
signals: list[Signal] = []
|
|
103
|
+
seen: set[tuple[str, str]] = set()
|
|
104
|
+
for later in commits:
|
|
105
|
+
for target in agent:
|
|
106
|
+
if target.sha == later.sha or later.ts <= target.ts:
|
|
107
|
+
continue
|
|
108
|
+
if later.ts - target.ts > REVERT_WINDOW:
|
|
109
|
+
continue
|
|
110
|
+
explicit = _explicitly_reverts(later, target)
|
|
111
|
+
removed = {} if explicit else _deletes_most_of(later, added[target.sha])
|
|
112
|
+
if not explicit and not removed:
|
|
113
|
+
continue
|
|
114
|
+
key = (later.sha, target.sha)
|
|
115
|
+
if key in seen:
|
|
116
|
+
continue
|
|
117
|
+
seen.add(key)
|
|
118
|
+
excerpt = _added_excerpt(target) if explicit else _fix_excerpt(later, removed)
|
|
119
|
+
signals.append(
|
|
120
|
+
Signal(
|
|
121
|
+
id=f"revert-{target.sha[:7]}-{later.sha[:7]}",
|
|
122
|
+
kind="revert",
|
|
123
|
+
commit_sha=target.sha,
|
|
124
|
+
files=[f.path for f in target.files] if explicit else list(removed),
|
|
125
|
+
diff_excerpt=excerpt,
|
|
126
|
+
text=f"{target.subject} -> {later.subject}",
|
|
127
|
+
author_is_agent=True,
|
|
128
|
+
score=1.0 if explicit else 0.6,
|
|
129
|
+
ts=later.ts,
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
return signals
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def detect_refixes(commits: list[Commit]) -> list[Signal]:
|
|
136
|
+
"""Human commits that remove lines an agent commit added, in the same place, within 48h."""
|
|
137
|
+
agent = [c for c in commits if c.is_agent]
|
|
138
|
+
added = {c.sha: _Added(c) for c in agent}
|
|
139
|
+
signals: list[Signal] = []
|
|
140
|
+
for later in commits:
|
|
141
|
+
if later.is_agent:
|
|
142
|
+
continue
|
|
143
|
+
for target in agent:
|
|
144
|
+
if later.ts <= target.ts or later.ts - target.ts > REFIX_WINDOW:
|
|
145
|
+
continue
|
|
146
|
+
if _explicitly_reverts(later, target):
|
|
147
|
+
continue # already a revert signal
|
|
148
|
+
removed = _removed_agent_lines(later, added[target.sha])
|
|
149
|
+
shared = {p: removed[p] for p in _overlapping_files(later, target) if p in removed}
|
|
150
|
+
if not shared:
|
|
151
|
+
continue
|
|
152
|
+
signals.append(
|
|
153
|
+
Signal(
|
|
154
|
+
id=f"refix-{target.sha[:7]}-{later.sha[:7]}",
|
|
155
|
+
kind="refix",
|
|
156
|
+
commit_sha=target.sha,
|
|
157
|
+
files=list(shared),
|
|
158
|
+
diff_excerpt=_fix_excerpt(later, shared),
|
|
159
|
+
text=f"{target.subject} -> {later.subject}",
|
|
160
|
+
author_is_agent=True,
|
|
161
|
+
score=0.8,
|
|
162
|
+
ts=later.ts,
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
return signals
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _pair_key(signal: Signal) -> str:
|
|
169
|
+
"""Revert and re-fix ids end in `<agent7>-<later7>`; anything else is unique by id."""
|
|
170
|
+
return signal.id.split("-", 1)[1] if signal.kind in ("revert", "refix") else signal.id
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def dedupe_signals(signals: list[Signal]) -> list[Signal]:
|
|
174
|
+
"""One signal per (agent commit, later commit) pair, keeping the most confident.
|
|
175
|
+
|
|
176
|
+
A human replacing the line an agent just wrote also "deletes most of" that commit, so the
|
|
177
|
+
revert and re-fix detectors can both fire on one event.
|
|
178
|
+
"""
|
|
179
|
+
best: dict[str, Signal] = {}
|
|
180
|
+
for s in signals:
|
|
181
|
+
key = _pair_key(s)
|
|
182
|
+
if key not in best or s.score > best[key].score:
|
|
183
|
+
best[key] = s
|
|
184
|
+
return [s for s in signals if best[_pair_key(s)] is s]
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _explicitly_reverts(later: Commit, target: Commit) -> bool:
|
|
188
|
+
return any(target.sha.startswith(m) for m in _REVERTS_SHA.findall(later.body))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _deletes_most_of(later: Commit, added: _Added) -> dict[str, list[str]]:
|
|
192
|
+
"""The removed agent lines per file when `later` removes at least half of them, else {}."""
|
|
193
|
+
if added.total == 0:
|
|
194
|
+
return {}
|
|
195
|
+
removed = _removed_agent_lines(later, added)
|
|
196
|
+
matched = sum(len(v) for v in removed.values())
|
|
197
|
+
return removed if matched >= MOSTLY_DELETED * added.total else {}
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _overlapping_files(later: Commit, target: Commit) -> list[str]:
|
|
201
|
+
"""Files where `later` edits the place `target` wrote.
|
|
202
|
+
|
|
203
|
+
Line numbers are compared directly, so commits between the two that shift lines can cause
|
|
204
|
+
misses. Content matching (above) is what keeps false matches out.
|
|
205
|
+
"""
|
|
206
|
+
theirs = {f.path: f for f in target.files}
|
|
207
|
+
return [f.path for f in later.files if f.path in theirs and _ranges_overlap(f, theirs[f.path])]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _ranges_overlap(later: FileChange, target: FileChange) -> bool:
|
|
211
|
+
return any(
|
|
212
|
+
a_lo <= b_hi and b_lo <= a_hi
|
|
213
|
+
for a_lo, a_hi in later.old_ranges
|
|
214
|
+
for b_lo, b_hi in target.new_ranges
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# Opening words of a message that rejects or corrects the agent ("no, use X", "don't touch Y").
|
|
219
|
+
_CORRECTION = re.compile(
|
|
220
|
+
r"^\s*(no\b[,.!:]?|nope\b|don'?t\b|do not\b|stop\b|wrong\b|that'?s (not|wrong)\b"
|
|
221
|
+
r"|not what i\b|undo\b|revert\b|why did you\b|instead\b"
|
|
222
|
+
r"|you (should|shouldn'?t|didn'?t|forgot)\b)",
|
|
223
|
+
re.I,
|
|
224
|
+
)
|
|
225
|
+
MAX_CORRECTION_CHARS = 400 # long messages are new tasks, not corrections
|
|
226
|
+
# Tags the harness injects into user turns; these are not the human talking.
|
|
227
|
+
_HARNESS_TEXT = re.compile(r"^\s*<[a-z-]+>|^\s*\[Request interrupted", re.I)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def detect_corrections(messages: list[SessionMessage]) -> list[Signal]:
|
|
231
|
+
"""User messages that reject or correct the agent's previous turn."""
|
|
232
|
+
signals: list[Signal] = []
|
|
233
|
+
seen_assistant: set[str] = set()
|
|
234
|
+
for m in sorted(messages, key=lambda m: m.ts):
|
|
235
|
+
if m.role == "assistant":
|
|
236
|
+
seen_assistant.add(m.session_id)
|
|
237
|
+
continue
|
|
238
|
+
if m.session_id not in seen_assistant:
|
|
239
|
+
continue # the opening message of a session is the task itself
|
|
240
|
+
if len(m.text) > MAX_CORRECTION_CHARS or _HARNESS_TEXT.match(m.text):
|
|
241
|
+
continue
|
|
242
|
+
if not _CORRECTION.match(m.text):
|
|
243
|
+
continue
|
|
244
|
+
signals.append(
|
|
245
|
+
Signal(
|
|
246
|
+
id=f"correction-{m.uuid[:8]}",
|
|
247
|
+
kind="correction",
|
|
248
|
+
commit_sha="", # evidence is the session message, not a commit
|
|
249
|
+
files=[],
|
|
250
|
+
text=m.text,
|
|
251
|
+
author_is_agent=True,
|
|
252
|
+
score=0.5,
|
|
253
|
+
ts=m.ts,
|
|
254
|
+
)
|
|
255
|
+
)
|
|
256
|
+
return signals
|
rulewright/mine/score.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Score step: weight each signal's detector confidence by recency, drop weak ones."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from rulewright.mine.signals import Signal
|
|
6
|
+
|
|
7
|
+
HALF_LIFE_DAYS = 180.0
|
|
8
|
+
THRESHOLD = 0.2
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def recency_weight(ts: datetime, now: datetime, half_life_days: float = HALF_LIFE_DAYS) -> float:
|
|
12
|
+
age_days = max((now - ts).total_seconds() / 86400, 0.0)
|
|
13
|
+
return float(0.5 ** (age_days / half_life_days))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def score_signals(
|
|
17
|
+
signals: list[Signal],
|
|
18
|
+
now: datetime,
|
|
19
|
+
threshold: float = THRESHOLD,
|
|
20
|
+
half_life_days: float = HALF_LIFE_DAYS,
|
|
21
|
+
) -> list[Signal]:
|
|
22
|
+
"""Detectors set `score` to their confidence; apply this once per pipeline run."""
|
|
23
|
+
out = []
|
|
24
|
+
for s in signals:
|
|
25
|
+
weighted = s.score * recency_weight(s.ts, now, half_life_days)
|
|
26
|
+
if weighted >= threshold:
|
|
27
|
+
out.append(s.model_copy(update={"score": weighted}))
|
|
28
|
+
return out
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel
|
|
5
|
+
|
|
6
|
+
SignalKind = Literal["revert", "refix", "correction", "review"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Signal(BaseModel):
|
|
10
|
+
"""One observed agent mistake. Text fields are untrusted data."""
|
|
11
|
+
|
|
12
|
+
id: str
|
|
13
|
+
kind: SignalKind
|
|
14
|
+
commit_sha: str
|
|
15
|
+
files: list[str]
|
|
16
|
+
diff_excerpt: str = ""
|
|
17
|
+
text: str
|
|
18
|
+
author_is_agent: bool
|
|
19
|
+
score: float
|
|
20
|
+
ts: datetime
|
rulewright/pipeline.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""The scan pipeline: history in, evidence-backed candidate rules out. Writes nothing."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
|
|
9
|
+
from rulewright.cluster.embed import Embedder, embed_signals
|
|
10
|
+
from rulewright.cluster.group import (
|
|
11
|
+
COHERENCE_FLOOR,
|
|
12
|
+
Cluster,
|
|
13
|
+
cluster_signals,
|
|
14
|
+
coherence,
|
|
15
|
+
cold_start_message,
|
|
16
|
+
)
|
|
17
|
+
from rulewright.ingest.git_history import read_commits
|
|
18
|
+
from rulewright.ingest.sessions.base import SessionSource
|
|
19
|
+
from rulewright.mine.detectors import (
|
|
20
|
+
dedupe_signals,
|
|
21
|
+
detect_corrections,
|
|
22
|
+
detect_refixes,
|
|
23
|
+
detect_reverts,
|
|
24
|
+
)
|
|
25
|
+
from rulewright.mine.score import score_signals
|
|
26
|
+
from rulewright.mine.signals import Signal
|
|
27
|
+
from rulewright.pr import commit_url
|
|
28
|
+
from rulewright.propose.llm import CostEstimate, LLMClient, estimate_cost
|
|
29
|
+
from rulewright.propose.writer import Proposals, build_prompt, propose_rules
|
|
30
|
+
from rulewright.rules.source import RulesFile
|
|
31
|
+
from rulewright.rules.stale import StaleFinding, stale_findings
|
|
32
|
+
from rulewright.store import open_store
|
|
33
|
+
|
|
34
|
+
MAX_RULES = 10 # spec: print 5-10 proposed rules
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ScanResult(BaseModel):
|
|
38
|
+
commits_read: int
|
|
39
|
+
agent_commits: int
|
|
40
|
+
signals: list[Signal]
|
|
41
|
+
clusters: list[Cluster]
|
|
42
|
+
proposals: Proposals | None # None when no rules were requested from an LLM
|
|
43
|
+
stale: list[StaleFinding]
|
|
44
|
+
cold_start: str | None
|
|
45
|
+
notes: list[str]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def select_clusters(
|
|
49
|
+
clusters: list[Cluster], signals: list[Signal], known_rule_ids: set[str]
|
|
50
|
+
) -> tuple[list[Cluster], int]:
|
|
51
|
+
"""The clusters worth an LLM call, best first, and how many were skipped as incoherent.
|
|
52
|
+
|
|
53
|
+
Ranked by how much their changes have in common, then by distinct commits (many signals from
|
|
54
|
+
one commit are one occurrence), so the limit keeps the most promising clusters, not the
|
|
55
|
+
largest.
|
|
56
|
+
"""
|
|
57
|
+
by_id = {s.id: s for s in signals}
|
|
58
|
+
scored = []
|
|
59
|
+
for c in clusters:
|
|
60
|
+
if "r-" + c.id.removeprefix("c-") in known_rule_ids:
|
|
61
|
+
continue
|
|
62
|
+
members = [by_id[i] for i in c.signal_ids if i in by_id]
|
|
63
|
+
commits = len({s.commit_sha or s.id for s in members})
|
|
64
|
+
scored.append((coherence(members), commits, c))
|
|
65
|
+
keep = [t for t in scored if t[0] >= COHERENCE_FLOOR]
|
|
66
|
+
keep.sort(key=lambda t: (-round(t[0], 2), -t[1], -t[2].size, t[2].id))
|
|
67
|
+
return [c for _, _, c in keep][:MAX_RULES], len(scored) - len(keep)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def run_scan(
|
|
71
|
+
repo: Path,
|
|
72
|
+
commits: int,
|
|
73
|
+
embedder: Embedder,
|
|
74
|
+
client: LLMClient | None,
|
|
75
|
+
existing: RulesFile,
|
|
76
|
+
now: datetime,
|
|
77
|
+
confirm: Callable[[CostEstimate], bool] = lambda _estimate: True,
|
|
78
|
+
sessions: list[SessionSource] | None = None,
|
|
79
|
+
progress: Callable[[int, int], None] | None = None,
|
|
80
|
+
judge: bool = True,
|
|
81
|
+
stage: Callable[[str], None] = lambda _name: None,
|
|
82
|
+
) -> ScanResult:
|
|
83
|
+
stage(f"Reading {commits} commits of git history")
|
|
84
|
+
history = read_commits(repo, commits)
|
|
85
|
+
agent_commits = sum(1 for c in history if c.is_agent)
|
|
86
|
+
notes: list[str] = []
|
|
87
|
+
|
|
88
|
+
stage(f"Looking for reverts and re-fixes in {len(history)} commits")
|
|
89
|
+
signals = detect_reverts(history) + detect_refixes(history)
|
|
90
|
+
for source in sessions or []:
|
|
91
|
+
stage("Reading agent session logs")
|
|
92
|
+
signals += detect_corrections(source.read(repo))
|
|
93
|
+
signals = score_signals(dedupe_signals(signals), now)
|
|
94
|
+
|
|
95
|
+
clusters: list[Cluster] = []
|
|
96
|
+
if signals:
|
|
97
|
+
stage(f"Embedding and clustering {len(signals)} signals (first run loads the model)")
|
|
98
|
+
conn = open_store(repo)
|
|
99
|
+
try:
|
|
100
|
+
clusters = cluster_signals(signals, embed_signals(signals, embedder, conn))
|
|
101
|
+
finally:
|
|
102
|
+
conn.close()
|
|
103
|
+
fresh, incoherent = select_clusters(clusters, signals, {r.id for r in existing.rules})
|
|
104
|
+
if incoherent:
|
|
105
|
+
notes.append(
|
|
106
|
+
f"Skipped {incoherent} cluster(s) whose changes share little in common "
|
|
107
|
+
"(unrelated edits that only look alike)."
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
cold_start = cold_start_message(len(clusters), commits)
|
|
111
|
+
if agent_commits == 0:
|
|
112
|
+
notes.append(
|
|
113
|
+
"No agent-authored commits found (looked for Co-Authored-By trailers and bot authors)."
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
proposals: Proposals | None = None
|
|
117
|
+
if cold_start is None and fresh and client is None:
|
|
118
|
+
notes.append(
|
|
119
|
+
"No LLM available: start Ollama or set ANTHROPIC_API_KEY / OPENAI_API_KEY to "
|
|
120
|
+
"turn these clusters into rules."
|
|
121
|
+
)
|
|
122
|
+
elif cold_start is None and fresh and client is not None:
|
|
123
|
+
by_id = {s.id: s for s in signals}
|
|
124
|
+
prompts = [
|
|
125
|
+
build_prompt(c, [by_id[i] for i in c.signal_ids if i in by_id]) for c in fresh
|
|
126
|
+
]
|
|
127
|
+
if judge: # the second call per rule is about as large as the first
|
|
128
|
+
prompts = prompts + prompts
|
|
129
|
+
if confirm(estimate_cost(client.model, prompts)):
|
|
130
|
+
proposals = propose_rules(fresh, signals, repo, client, progress, judge)
|
|
131
|
+
for rule in proposals.rules:
|
|
132
|
+
for e in rule.evidence:
|
|
133
|
+
e.url = commit_url(repo, e.commit_sha) if e.commit_sha else ""
|
|
134
|
+
else:
|
|
135
|
+
notes.append("Cancelled before any LLM call; no rules were proposed.")
|
|
136
|
+
|
|
137
|
+
return ScanResult(
|
|
138
|
+
commits_read=len(history),
|
|
139
|
+
agent_commits=agent_commits,
|
|
140
|
+
signals=signals,
|
|
141
|
+
clusters=clusters,
|
|
142
|
+
proposals=proposals,
|
|
143
|
+
stale=stale_findings(existing.rules, repo, signals, now),
|
|
144
|
+
cold_start=cold_start,
|
|
145
|
+
notes=notes,
|
|
146
|
+
)
|
rulewright/pr.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""`scan --pr`: put accepted rules on a new branch and open a pull request for review.
|
|
2
|
+
|
|
3
|
+
The pull request is the approval step: the user reviews and merges it (or not). The working
|
|
4
|
+
branch is restored afterwards and nothing is changed if a target file has broken markers.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
import shutil
|
|
9
|
+
import subprocess
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel
|
|
15
|
+
|
|
16
|
+
from rulewright.rules.emit import plan_emit, write_changes
|
|
17
|
+
from rulewright.rules.model import Rule
|
|
18
|
+
from rulewright.rules.source import FILENAME, RulesFile, save_rules
|
|
19
|
+
|
|
20
|
+
_GITHUB = re.compile(r"github\.com[:/]([^/\s]+)/([^/\s]+?)(?:\.git)?/?$")
|
|
21
|
+
Gh = Callable[[list[str], Path], str]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class PRError(Exception):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PRResult(BaseModel):
|
|
29
|
+
branch: str
|
|
30
|
+
pushed: bool
|
|
31
|
+
url: str | None
|
|
32
|
+
notes: list[str]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _git(repo: Path, *args: str, check: bool = True) -> str:
|
|
36
|
+
proc = subprocess.run(
|
|
37
|
+
["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8"
|
|
38
|
+
)
|
|
39
|
+
if check and proc.returncode != 0:
|
|
40
|
+
raise PRError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
|
|
41
|
+
return proc.stdout.strip()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def github_slug(repo: Path) -> str | None:
|
|
45
|
+
"""`owner/name` of the origin remote when it is on GitHub."""
|
|
46
|
+
m = _GITHUB.search(_git(repo, "remote", "get-url", "origin", check=False))
|
|
47
|
+
return f"{m.group(1)}/{m.group(2)}" if m else None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def commit_url(repo: Path, sha: str) -> str:
|
|
51
|
+
slug = github_slug(repo)
|
|
52
|
+
return f"https://github.com/{slug}/commit/{sha}" if slug else ""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def pr_body(rules: list[Rule], repo: Path) -> str:
|
|
56
|
+
lines = ["rulewright proposes these rules. Each cites the evidence behind it.", ""]
|
|
57
|
+
for r in rules:
|
|
58
|
+
lines.append(f"### {r.id}\n> {' '.join(r.text.split())}\n")
|
|
59
|
+
lines.append("Scope: " + ", ".join(f"`{s}`" for s in r.scope) + "\n")
|
|
60
|
+
lines.append("Evidence:")
|
|
61
|
+
for e in r.evidence:
|
|
62
|
+
ref = e.commit_sha or e.signal_id
|
|
63
|
+
url = commit_url(repo, e.commit_sha) if e.commit_sha else ""
|
|
64
|
+
lines.append(f"- [{ref[:7]}]({url})" if url else f"- {ref[:12]}")
|
|
65
|
+
lines.append("")
|
|
66
|
+
return "\n".join(lines)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _default_gh(args: list[str], cwd: Path) -> str:
|
|
70
|
+
gh = shutil.which("gh")
|
|
71
|
+
if gh is None:
|
|
72
|
+
raise PRError("the GitHub CLI (gh) is not installed")
|
|
73
|
+
proc = subprocess.run([gh, *args], cwd=cwd, capture_output=True, text=True, encoding="utf-8")
|
|
74
|
+
if proc.returncode != 0:
|
|
75
|
+
raise PRError(f"gh {' '.join(args[:2])} failed: {proc.stderr.strip()}")
|
|
76
|
+
return proc.stdout.strip()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def create_pr(
|
|
80
|
+
repo: Path, rules: list[Rule], rules_file: RulesFile, now: datetime, gh: Gh = _default_gh
|
|
81
|
+
) -> PRResult:
|
|
82
|
+
"""Commit `rules` (as accepted) to a new branch, push it and open a pull request."""
|
|
83
|
+
if not rules:
|
|
84
|
+
raise PRError("no rules to propose")
|
|
85
|
+
updated = rules_file.model_copy(deep=True)
|
|
86
|
+
for rule in rules:
|
|
87
|
+
updated.add(rule, "accepted")
|
|
88
|
+
changes = plan_emit(repo, updated) # raises before any branch exists if markers are broken
|
|
89
|
+
touched = [FILENAME, *(str(c.path.relative_to(repo)) for c in changes)]
|
|
90
|
+
dirty = _git(repo, "status", "--porcelain", "--", *touched)
|
|
91
|
+
if dirty:
|
|
92
|
+
raise PRError(f"uncommitted changes in files rulewright would edit:\n{dirty}")
|
|
93
|
+
|
|
94
|
+
original = _git(repo, "rev-parse", "--abbrev-ref", "HEAD")
|
|
95
|
+
branch = f"rulewright/rules-{now:%Y%m%d-%H%M%S}"
|
|
96
|
+
_git(repo, "switch", "-q", "-c", branch)
|
|
97
|
+
notes: list[str] = []
|
|
98
|
+
pushed, url = False, None
|
|
99
|
+
try:
|
|
100
|
+
save_rules(repo, updated)
|
|
101
|
+
write_changes(changes)
|
|
102
|
+
_git(repo, "add", "--", *touched)
|
|
103
|
+
_git(repo, "commit", "-q", "-m", f"Add {len(rules)} rule(s) learned by rulewright")
|
|
104
|
+
if _git(repo, "remote", check=False).split():
|
|
105
|
+
_git(repo, "push", "-q", "-u", "origin", branch)
|
|
106
|
+
pushed = True
|
|
107
|
+
try:
|
|
108
|
+
body = pr_body(rules, repo)
|
|
109
|
+
body_file = repo / ".rulewright" / "pr-body.md"
|
|
110
|
+
body_file.parent.mkdir(exist_ok=True)
|
|
111
|
+
body_file.write_text(body, encoding="utf-8")
|
|
112
|
+
url = gh(
|
|
113
|
+
["pr", "create", "--title", f"rulewright: {len(rules)} proposed rule(s)",
|
|
114
|
+
"--body-file", str(body_file)],
|
|
115
|
+
repo,
|
|
116
|
+
) # fmt: skip
|
|
117
|
+
except PRError as e:
|
|
118
|
+
notes.append(f"Branch pushed but no pull request opened: {e}")
|
|
119
|
+
else:
|
|
120
|
+
notes.append("No git remote; the branch is local only.")
|
|
121
|
+
finally:
|
|
122
|
+
_git(repo, "switch", "-q", original, check=False)
|
|
123
|
+
return PRResult(branch=branch, pushed=pushed, url=url, notes=notes)
|
|
File without changes
|