trace2rules 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trace2rules/__init__.py +1 -0
- trace2rules/cli.py +543 -0
- trace2rules/cluster.py +348 -0
- trace2rules/db.py +84 -0
- trace2rules/detectors.py +183 -0
- trace2rules/ingest.py +717 -0
- trace2rules/labels.py +431 -0
- trace2rules/models.py +33 -0
- trace2rules/patterns.py +159 -0
- trace2rules/sources/__init__.py +1 -0
- trace2rules/sources/agent_trajectory.py +456 -0
- trace2rules/sources/common.py +99 -0
- trace2rules/sources/opencode.py +154 -0
- trace2rules/sources/swe_chat.py +117 -0
- trace2rules/synthesize.py +538 -0
- trace2rules-0.1.0.dist-info/METADATA +323 -0
- trace2rules-0.1.0.dist-info/RECORD +20 -0
- trace2rules-0.1.0.dist-info/WHEEL +4 -0
- trace2rules-0.1.0.dist-info/entry_points.txt +3 -0
- trace2rules-0.1.0.dist-info/licenses/LICENSE +21 -0
trace2rules/cluster.py
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
import duckdb
|
|
7
|
+
from sklearn.cluster import AgglomerativeClustering
|
|
8
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
9
|
+
from sklearn.metrics.pairwise import cosine_distances
|
|
10
|
+
|
|
11
|
+
from .detectors import Finding, context_blasts, hotspots, recovery_loops
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class _Window:
|
|
16
|
+
repo: str
|
|
17
|
+
encoded: str
|
|
18
|
+
sessions: set[str] = field(default_factory=set)
|
|
19
|
+
wasted_tokens: int = 0
|
|
20
|
+
examples: list[str] = field(default_factory=list)
|
|
21
|
+
tokens: tuple[str, ...] = field(default_factory=tuple)
|
|
22
|
+
keys: tuple[tuple[str, str, str], ...] = field(default_factory=tuple)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _encode(window: list[tuple]) -> str:
|
|
26
|
+
parts = []
|
|
27
|
+
for position, event in enumerate(window, start=1):
|
|
28
|
+
_session, _repo, _step, _tool, action, verb, object_id, outcome = event[:8]
|
|
29
|
+
parts.append(f"s{position}:{action}:{verb}:{object_id}:{outcome}")
|
|
30
|
+
return " | ".join(parts)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _tokens(window: list[tuple]) -> tuple[str, ...]:
|
|
34
|
+
return tuple(
|
|
35
|
+
f"{event[4]}:{event[5]}:{event[6]}:{event[7]}" for event in window
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _example(window: list[tuple]) -> str:
|
|
40
|
+
return " -> ".join(
|
|
41
|
+
f"{event[4]}:{(event[11] or event[10]).replace(chr(0), ' ')[:70]}" for event in window
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def friction_windows(
|
|
46
|
+
con: duckdb.DuckDBPyConnection,
|
|
47
|
+
min_session_count: int = 2,
|
|
48
|
+
lengths: tuple[int, ...] = (3, 4),
|
|
49
|
+
) -> list[_Window]:
|
|
50
|
+
hotspot_targets: dict[str, set[str]] = defaultdict(set)
|
|
51
|
+
for finding in hotspots(con):
|
|
52
|
+
hotspot_targets[finding.repo].add(finding.title)
|
|
53
|
+
blast_targets = {(f.repo, f.title) for f in context_blasts(con)}
|
|
54
|
+
|
|
55
|
+
rows = con.execute(
|
|
56
|
+
"""
|
|
57
|
+
SELECT session_id, repo, step_idx, tool_name, action_type, verb, object_id,
|
|
58
|
+
outcome, error_signature, approx_out_tokens, target, raw_target,
|
|
59
|
+
segment_idx
|
|
60
|
+
FROM tool_events
|
|
61
|
+
WHERE target <> '' AND action_type <> 'other'
|
|
62
|
+
ORDER BY session_id, step_idx
|
|
63
|
+
"""
|
|
64
|
+
).fetchall()
|
|
65
|
+
sessions: dict[str, list[tuple]] = defaultdict(list)
|
|
66
|
+
for row in rows:
|
|
67
|
+
sessions[row[0]].append(row)
|
|
68
|
+
|
|
69
|
+
aggregates: dict[tuple[str, str], _Window] = {}
|
|
70
|
+
for session_id, events in sessions.items():
|
|
71
|
+
for k in lengths:
|
|
72
|
+
for start in range(len(events) - k + 1):
|
|
73
|
+
window = events[start : start + k]
|
|
74
|
+
if len({event[12] for event in window}) > 1:
|
|
75
|
+
continue
|
|
76
|
+
relevant = any(event[7] == "err" for event in window)
|
|
77
|
+
if not relevant:
|
|
78
|
+
relevant = any(
|
|
79
|
+
event[4] in ("read", "search") and event[10] in hotspot_targets[event[1]]
|
|
80
|
+
for event in window
|
|
81
|
+
)
|
|
82
|
+
if not relevant:
|
|
83
|
+
relevant = any(
|
|
84
|
+
event[9] >= 1500
|
|
85
|
+
or (event[1], event[10]) in blast_targets
|
|
86
|
+
for event in window
|
|
87
|
+
)
|
|
88
|
+
if not relevant:
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
encoded = _encode(window)
|
|
92
|
+
key = (window[0][1], encoded)
|
|
93
|
+
agg = aggregates.get(key)
|
|
94
|
+
if agg is None:
|
|
95
|
+
agg = aggregates[key] = _Window(
|
|
96
|
+
repo=window[0][1],
|
|
97
|
+
encoded=encoded,
|
|
98
|
+
tokens=_tokens(window),
|
|
99
|
+
keys=tuple(
|
|
100
|
+
(event[3], event[4], event[10]) for event in window
|
|
101
|
+
),
|
|
102
|
+
)
|
|
103
|
+
agg.sessions.add(session_id)
|
|
104
|
+
agg.wasted_tokens += sum(
|
|
105
|
+
event[9]
|
|
106
|
+
for event in window
|
|
107
|
+
if event[7] == "err"
|
|
108
|
+
or event[9] >= 1500
|
|
109
|
+
or (event[4] in ("read", "search") and event[10] in hotspot_targets[event[1]])
|
|
110
|
+
)
|
|
111
|
+
example = _example(window)
|
|
112
|
+
if len(agg.examples) < 3 and example not in agg.examples:
|
|
113
|
+
agg.examples.append(example)
|
|
114
|
+
|
|
115
|
+
return [w for w in aggregates.values() if len(w.sessions) >= 1]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def exact_sequences(
|
|
119
|
+
con: duckdb.DuckDBPyConnection,
|
|
120
|
+
length: int = 3,
|
|
121
|
+
min_sessions: int = 2,
|
|
122
|
+
top: int = 25,
|
|
123
|
+
require_error: bool = True,
|
|
124
|
+
min_instances: int = 2,
|
|
125
|
+
min_rework: int = 0,
|
|
126
|
+
min_lift: float = 1.5,
|
|
127
|
+
retry_weight: int = 1000,
|
|
128
|
+
) -> list[Finding]:
|
|
129
|
+
repo_sessions = dict(
|
|
130
|
+
con.execute("SELECT repo, count(*) FROM sessions GROUP BY repo").fetchall()
|
|
131
|
+
)
|
|
132
|
+
repo_instances = dict(
|
|
133
|
+
con.execute(
|
|
134
|
+
"SELECT repo, count(DISTINCT instance_id) FROM sessions GROUP BY repo"
|
|
135
|
+
).fetchall()
|
|
136
|
+
)
|
|
137
|
+
session_instances = dict(
|
|
138
|
+
con.execute("SELECT session_id, instance_id FROM sessions").fetchall()
|
|
139
|
+
)
|
|
140
|
+
session_rework: dict[str, int] = defaultdict(int)
|
|
141
|
+
instance_sessions: dict[str, set[str]] = defaultdict(set)
|
|
142
|
+
for session_id, instance_id in session_instances.items():
|
|
143
|
+
instance_sessions[instance_id].add(session_id)
|
|
144
|
+
for session_id, _verb, _object_id, errors in con.execute(
|
|
145
|
+
"""
|
|
146
|
+
SELECT session_id, verb, object_id, count(*) FILTER (WHERE outcome = 'err')
|
|
147
|
+
FROM tool_events
|
|
148
|
+
WHERE outcome = 'err' AND object_id <> ''
|
|
149
|
+
GROUP BY 1, 2, 3
|
|
150
|
+
"""
|
|
151
|
+
).fetchall():
|
|
152
|
+
if errors > 1:
|
|
153
|
+
session_rework[session_id] += errors - 1
|
|
154
|
+
|
|
155
|
+
session_setup: dict[str, int] = {}
|
|
156
|
+
grouped = defaultdict(list)
|
|
157
|
+
for row in con.execute(
|
|
158
|
+
"""
|
|
159
|
+
SELECT session_id, step_idx, action_type, verb, outcome, approx_out_tokens
|
|
160
|
+
FROM tool_events
|
|
161
|
+
WHERE action_type <> 'other'
|
|
162
|
+
ORDER BY session_id, step_idx
|
|
163
|
+
"""
|
|
164
|
+
).fetchall():
|
|
165
|
+
grouped[row[0]].append(row)
|
|
166
|
+
for session_id, events in grouped.items():
|
|
167
|
+
setup = sum(event[5] for event in events)
|
|
168
|
+
for index, event in enumerate(events):
|
|
169
|
+
if (event[2] == "edit" and event[4] == "ok") or (
|
|
170
|
+
event[3] == "pytest" and event[4] == "ok"
|
|
171
|
+
):
|
|
172
|
+
setup = sum(item[5] for item in events[:index])
|
|
173
|
+
break
|
|
174
|
+
session_setup[session_id] = setup
|
|
175
|
+
token_sessions: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
|
|
176
|
+
for session_id, repo, action, verb, object_id, outcome in con.execute(
|
|
177
|
+
"""
|
|
178
|
+
SELECT session_id, repo, action_type, verb, object_id, outcome
|
|
179
|
+
FROM tool_events
|
|
180
|
+
WHERE target <> '' AND action_type <> 'other'
|
|
181
|
+
"""
|
|
182
|
+
).fetchall():
|
|
183
|
+
token_sessions[repo][f"{action}:{verb}:{object_id}:{outcome}"].add(session_id)
|
|
184
|
+
|
|
185
|
+
scored: list[tuple[float, Finding]] = []
|
|
186
|
+
for window in friction_windows(con, lengths=(length,)):
|
|
187
|
+
if require_error and ":err" not in window.encoded:
|
|
188
|
+
continue
|
|
189
|
+
if len(window.sessions) < min_sessions:
|
|
190
|
+
continue
|
|
191
|
+
instances = {
|
|
192
|
+
session_instances.get(session) for session in window.sessions
|
|
193
|
+
} - {None}
|
|
194
|
+
required_instances = min(
|
|
195
|
+
min_instances, repo_instances.get(window.repo, min_instances)
|
|
196
|
+
)
|
|
197
|
+
if len(instances) < required_instances:
|
|
198
|
+
continue
|
|
199
|
+
n = repo_sessions.get(window.repo, 0)
|
|
200
|
+
expected = float(n)
|
|
201
|
+
for token in window.tokens:
|
|
202
|
+
expected *= len(token_sessions[window.repo].get(token, ())) / n if n else 0
|
|
203
|
+
lift = len(window.sessions) / expected if expected else 0.0
|
|
204
|
+
if lift < min_lift:
|
|
205
|
+
continue
|
|
206
|
+
rework = sum(session_rework.get(session, 0) for session in window.sessions)
|
|
207
|
+
if rework < min_rework:
|
|
208
|
+
continue
|
|
209
|
+
setup_tax = sum(session_setup.get(session, 0) for session in window.sessions)
|
|
210
|
+
paired = 0
|
|
211
|
+
rework_deltas: list[float] = []
|
|
212
|
+
setup_deltas: list[float] = []
|
|
213
|
+
for instance in instances:
|
|
214
|
+
present = [
|
|
215
|
+
session
|
|
216
|
+
for session in window.sessions
|
|
217
|
+
if session_instances.get(session) == instance
|
|
218
|
+
]
|
|
219
|
+
absent = instance_sessions.get(instance, set()) - set(present)
|
|
220
|
+
if not present or not absent:
|
|
221
|
+
continue
|
|
222
|
+
paired += 1
|
|
223
|
+
present_rework = sum(session_rework.get(s, 0) for s in present) / len(present)
|
|
224
|
+
absent_rework = sum(session_rework.get(s, 0) for s in absent) / len(absent)
|
|
225
|
+
present_setup = sum(session_setup.get(s, 0) for s in present) / len(present)
|
|
226
|
+
absent_setup = sum(session_setup.get(s, 0) for s in absent) / len(absent)
|
|
227
|
+
rework_deltas.append(present_rework - absent_rework)
|
|
228
|
+
setup_deltas.append(present_setup - absent_setup)
|
|
229
|
+
if paired < 2 and len(window.sessions) < 10:
|
|
230
|
+
continue
|
|
231
|
+
delta_rework = sum(rework_deltas) / len(rework_deltas) if rework_deltas else 0.0
|
|
232
|
+
delta_setup = sum(setup_deltas) / len(setup_deltas) if setup_deltas else 0.0
|
|
233
|
+
cost = window.wasted_tokens + rework * retry_weight
|
|
234
|
+
finding = Finding(
|
|
235
|
+
kind="exact",
|
|
236
|
+
repo=window.repo,
|
|
237
|
+
title=window.encoded,
|
|
238
|
+
sessions=len(window.sessions),
|
|
239
|
+
tokens=window.wasted_tokens,
|
|
240
|
+
detail=(
|
|
241
|
+
f"exact {length}-step; lift {lift:.1f}; cost {cost:,}; "
|
|
242
|
+
f"rework {rework}; setup {setup_tax:,}; paired {paired}; "
|
|
243
|
+
f"delta_rework {delta_rework:+.1f}; delta_setup {delta_setup:+,.0f}"
|
|
244
|
+
),
|
|
245
|
+
sequence=window.encoded,
|
|
246
|
+
variants=tuple(window.examples[:3]),
|
|
247
|
+
event_keys=window.keys,
|
|
248
|
+
lift=lift,
|
|
249
|
+
rework=rework,
|
|
250
|
+
setup_tax=setup_tax,
|
|
251
|
+
cost=cost,
|
|
252
|
+
paired_instances=paired,
|
|
253
|
+
delta_rework=delta_rework,
|
|
254
|
+
delta_setup=delta_setup,
|
|
255
|
+
)
|
|
256
|
+
scored.append((cost, finding))
|
|
257
|
+
scored.sort(key=lambda item: (item[0], item[1].sessions, item[1].lift), reverse=True)
|
|
258
|
+
return [finding for _, finding in scored[:top]]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _cluster_windows(windows: list[_Window], distance_threshold: float) -> list[Finding]:
|
|
262
|
+
if len(windows) < 2:
|
|
263
|
+
return []
|
|
264
|
+
|
|
265
|
+
corpus = [w.encoded for w in windows]
|
|
266
|
+
vectorizer = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), lowercase=False)
|
|
267
|
+
try:
|
|
268
|
+
matrix = vectorizer.fit_transform(corpus)
|
|
269
|
+
distances = cosine_distances(matrix)
|
|
270
|
+
except ValueError:
|
|
271
|
+
return []
|
|
272
|
+
|
|
273
|
+
labels = AgglomerativeClustering(
|
|
274
|
+
n_clusters=None,
|
|
275
|
+
distance_threshold=distance_threshold,
|
|
276
|
+
metric="precomputed",
|
|
277
|
+
linkage="average",
|
|
278
|
+
).fit_predict(distances)
|
|
279
|
+
|
|
280
|
+
groups: dict[int, list[_Window]] = defaultdict(list)
|
|
281
|
+
for window, label in zip(windows, labels):
|
|
282
|
+
groups[int(label)].append(window)
|
|
283
|
+
|
|
284
|
+
findings: list[Finding] = []
|
|
285
|
+
for members in groups.values():
|
|
286
|
+
session_ids: set[str] = set()
|
|
287
|
+
for member in members:
|
|
288
|
+
session_ids.update(member.sessions)
|
|
289
|
+
if len(session_ids) < 2:
|
|
290
|
+
continue
|
|
291
|
+
|
|
292
|
+
medoid = max(members, key=lambda m: (len(m.sessions), m.wasted_tokens))
|
|
293
|
+
variants: list[str] = []
|
|
294
|
+
for member in sorted(members, key=lambda m: len(m.sessions), reverse=True):
|
|
295
|
+
for example in member.examples:
|
|
296
|
+
if example not in variants:
|
|
297
|
+
variants.append(example)
|
|
298
|
+
if len(variants) == 3:
|
|
299
|
+
break
|
|
300
|
+
if len(variants) == 3:
|
|
301
|
+
break
|
|
302
|
+
|
|
303
|
+
findings.append(
|
|
304
|
+
Finding(
|
|
305
|
+
kind="cluster",
|
|
306
|
+
repo=medoid.repo,
|
|
307
|
+
title=medoid.encoded,
|
|
308
|
+
sessions=len(session_ids),
|
|
309
|
+
tokens=sum(member.wasted_tokens for member in members),
|
|
310
|
+
detail=f"{len(members)} distinct windows merged",
|
|
311
|
+
sequence=medoid.encoded,
|
|
312
|
+
variants=tuple(variants),
|
|
313
|
+
event_keys=medoid.keys,
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
return findings
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def cluster_friction(
|
|
320
|
+
con: duckdb.DuckDBPyConnection,
|
|
321
|
+
top: int | None = 25,
|
|
322
|
+
distance_threshold: float = 0.25,
|
|
323
|
+
) -> list[Finding]:
|
|
324
|
+
by_repo: dict[str, list[_Window]] = defaultdict(list)
|
|
325
|
+
for window in friction_windows(con):
|
|
326
|
+
by_repo[window.repo].append(window)
|
|
327
|
+
|
|
328
|
+
findings: list[Finding] = []
|
|
329
|
+
for repo_windows in by_repo.values():
|
|
330
|
+
if len(repo_windows) > 4000:
|
|
331
|
+
repo_windows = sorted(
|
|
332
|
+
repo_windows, key=lambda w: (len(w.sessions), w.wasted_tokens), reverse=True
|
|
333
|
+
)[:4000]
|
|
334
|
+
findings.extend(_cluster_windows(repo_windows, distance_threshold))
|
|
335
|
+
|
|
336
|
+
findings.sort(key=lambda f: (f.sessions, f.tokens), reverse=True)
|
|
337
|
+
return findings[:top] if top is not None else findings
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def mine(
|
|
341
|
+
con: duckdb.DuckDBPyConnection, top: int | None = 25
|
|
342
|
+
) -> dict[str, list[Finding]]:
|
|
343
|
+
return {
|
|
344
|
+
"hotspots": hotspots(con, limit=top),
|
|
345
|
+
"recovery": recovery_loops(con, limit=top),
|
|
346
|
+
"context": context_blasts(con, limit=top),
|
|
347
|
+
"clusters": cluster_friction(con, top=top),
|
|
348
|
+
}
|
trace2rules/db.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import duckdb
|
|
6
|
+
|
|
7
|
+
DB_PATH = Path("data/trace2rules.duckdb")
|
|
8
|
+
|
|
9
|
+
SCHEMA = """
|
|
10
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
11
|
+
session_id VARCHAR PRIMARY KEY,
|
|
12
|
+
repo VARCHAR NOT NULL,
|
|
13
|
+
instance_id VARCHAR NOT NULL,
|
|
14
|
+
resolved BOOLEAN NOT NULL,
|
|
15
|
+
total_turns INTEGER NOT NULL,
|
|
16
|
+
source VARCHAR DEFAULT '',
|
|
17
|
+
agent VARCHAR DEFAULT '',
|
|
18
|
+
success_score DOUBLE
|
|
19
|
+
);
|
|
20
|
+
|
|
21
|
+
CREATE TABLE IF NOT EXISTS tool_events (
|
|
22
|
+
event_id VARCHAR PRIMARY KEY,
|
|
23
|
+
session_id VARCHAR NOT NULL,
|
|
24
|
+
repo VARCHAR NOT NULL,
|
|
25
|
+
step_idx INTEGER NOT NULL,
|
|
26
|
+
segment_idx INTEGER NOT NULL DEFAULT 0,
|
|
27
|
+
tool_name VARCHAR NOT NULL,
|
|
28
|
+
action_type VARCHAR NOT NULL,
|
|
29
|
+
verb VARCHAR NOT NULL,
|
|
30
|
+
object_id VARCHAR NOT NULL,
|
|
31
|
+
target VARCHAR NOT NULL,
|
|
32
|
+
raw_target VARCHAR NOT NULL,
|
|
33
|
+
outcome VARCHAR NOT NULL,
|
|
34
|
+
exit_code INTEGER,
|
|
35
|
+
error_signature VARCHAR NOT NULL,
|
|
36
|
+
is_error BOOLEAN NOT NULL,
|
|
37
|
+
approx_out_tokens INTEGER NOT NULL,
|
|
38
|
+
obs_excerpt VARCHAR NOT NULL
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
CREATE TABLE IF NOT EXISTS target_labels (
|
|
42
|
+
tool_name VARCHAR NOT NULL,
|
|
43
|
+
action_type VARCHAR NOT NULL,
|
|
44
|
+
target VARCHAR NOT NULL,
|
|
45
|
+
intent VARCHAR NOT NULL,
|
|
46
|
+
intent_confidence DOUBLE NOT NULL,
|
|
47
|
+
object_type VARCHAR NOT NULL,
|
|
48
|
+
object_confidence DOUBLE NOT NULL,
|
|
49
|
+
model VARCHAR NOT NULL,
|
|
50
|
+
labeled_at TIMESTAMP NOT NULL DEFAULT current_timestamp,
|
|
51
|
+
PRIMARY KEY (tool_name, action_type, target)
|
|
52
|
+
);
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def connect(path: str | Path = DB_PATH) -> duckdb.DuckDBPyConnection:
|
|
57
|
+
path = Path(path)
|
|
58
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
59
|
+
con = duckdb.connect(
|
|
60
|
+
str(path),
|
|
61
|
+
config={
|
|
62
|
+
"memory_limit": "4500MB",
|
|
63
|
+
"threads": "1",
|
|
64
|
+
"preserve_insertion_order": "false",
|
|
65
|
+
},
|
|
66
|
+
)
|
|
67
|
+
con.execute(SCHEMA)
|
|
68
|
+
for ddl in (
|
|
69
|
+
"ALTER TABLE sessions ADD COLUMN source VARCHAR DEFAULT ''",
|
|
70
|
+
"ALTER TABLE sessions ADD COLUMN agent VARCHAR DEFAULT ''",
|
|
71
|
+
"ALTER TABLE sessions ADD COLUMN success_score DOUBLE",
|
|
72
|
+
"ALTER TABLE tool_events ADD COLUMN verb VARCHAR DEFAULT ''",
|
|
73
|
+
"ALTER TABLE tool_events ADD COLUMN object_id VARCHAR DEFAULT ''",
|
|
74
|
+
"ALTER TABLE tool_events ADD COLUMN outcome VARCHAR DEFAULT 'unknown'",
|
|
75
|
+
"ALTER TABLE tool_events ADD COLUMN exit_code INTEGER",
|
|
76
|
+
"ALTER TABLE tool_events ADD COLUMN error_signature VARCHAR DEFAULT ''",
|
|
77
|
+
"ALTER TABLE tool_events ADD COLUMN obs_excerpt VARCHAR DEFAULT ''",
|
|
78
|
+
"ALTER TABLE tool_events ADD COLUMN segment_idx INTEGER DEFAULT 0",
|
|
79
|
+
):
|
|
80
|
+
try:
|
|
81
|
+
con.execute(ddl)
|
|
82
|
+
except duckdb.Error:
|
|
83
|
+
pass
|
|
84
|
+
return con
|
trace2rules/detectors.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
import duckdb
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class Finding:
|
|
10
|
+
kind: str
|
|
11
|
+
repo: str
|
|
12
|
+
title: str
|
|
13
|
+
sessions: int
|
|
14
|
+
tokens: int
|
|
15
|
+
detail: str = ""
|
|
16
|
+
sequence: str = ""
|
|
17
|
+
variants: tuple[str, ...] = field(default_factory=tuple)
|
|
18
|
+
lift: float = 0.0
|
|
19
|
+
rework: int = 0
|
|
20
|
+
setup_tax: int = 0
|
|
21
|
+
cost: int = 0
|
|
22
|
+
paired_instances: int = 0
|
|
23
|
+
delta_rework: float = 0.0
|
|
24
|
+
delta_setup: float = 0.0
|
|
25
|
+
intent: str = ""
|
|
26
|
+
object_type: str = ""
|
|
27
|
+
intent_confidence: float = 0.0
|
|
28
|
+
object_confidence: float = 0.0
|
|
29
|
+
event_keys: tuple[tuple[str, str, str], ...] = field(default_factory=tuple)
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def evidence(self) -> str:
|
|
33
|
+
return f"{self.detail}; ~{self.tokens:,} tokens"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def session_summary(con: duckdb.DuckDBPyConnection) -> list[dict]:
|
|
37
|
+
rows = con.execute(
|
|
38
|
+
"""
|
|
39
|
+
SELECT s.repo,
|
|
40
|
+
count(*) AS sessions,
|
|
41
|
+
sum(CASE WHEN s.resolved THEN 1 ELSE 0 END) AS resolved,
|
|
42
|
+
count(*) FILTER (WHERE s.source NOT IN ('swe-chat', 'agent-trajectory')) AS known_resolution,
|
|
43
|
+
(SELECT count(*) FROM tool_events e WHERE e.repo = s.repo) AS events,
|
|
44
|
+
(SELECT sum(approx_out_tokens) FROM tool_events e WHERE e.repo = s.repo) AS tokens
|
|
45
|
+
FROM sessions s
|
|
46
|
+
GROUP BY s.repo
|
|
47
|
+
ORDER BY sessions DESC
|
|
48
|
+
"""
|
|
49
|
+
).fetchall()
|
|
50
|
+
return [
|
|
51
|
+
{
|
|
52
|
+
"repo": r[0],
|
|
53
|
+
"sessions": r[1],
|
|
54
|
+
"resolved": r[2],
|
|
55
|
+
"known_resolution": r[3],
|
|
56
|
+
"events": r[4] or 0,
|
|
57
|
+
"tokens": r[5] or 0,
|
|
58
|
+
}
|
|
59
|
+
for r in rows
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def hotspots(con: duckdb.DuckDBPyConnection, limit: int | None = None) -> list[Finding]:
|
|
64
|
+
query = """
|
|
65
|
+
WITH per_session AS (
|
|
66
|
+
SELECT repo, target, session_id,
|
|
67
|
+
sum(approx_out_tokens) FILTER (WHERE action_type IN ('read', 'search')) AS read_tokens,
|
|
68
|
+
bool_or(action_type = 'edit') AS was_edited
|
|
69
|
+
FROM tool_events
|
|
70
|
+
WHERE target NOT IN ('', '.')
|
|
71
|
+
AND target NOT LIKE '<%'
|
|
72
|
+
AND target NOT LIKE '% %'
|
|
73
|
+
AND target NOT LIKE '%>%'
|
|
74
|
+
AND target NOT LIKE '%;'
|
|
75
|
+
AND target NOT LIKE 'skill:%'
|
|
76
|
+
AND target NOT LIKE 'xd:%'
|
|
77
|
+
GROUP BY repo, target, session_id
|
|
78
|
+
),
|
|
79
|
+
agg AS (
|
|
80
|
+
SELECT repo, target,
|
|
81
|
+
count(*) AS sessions,
|
|
82
|
+
count(*) FILTER (WHERE NOT was_edited) AS read_only_sessions,
|
|
83
|
+
sum(CASE WHEN NOT was_edited THEN coalesce(read_tokens, 0) ELSE 0 END) AS wasted_tokens
|
|
84
|
+
FROM per_session
|
|
85
|
+
WHERE read_tokens IS NOT NULL
|
|
86
|
+
GROUP BY repo, target
|
|
87
|
+
)
|
|
88
|
+
SELECT a.repo, a.target, a.sessions, a.read_only_sessions,
|
|
89
|
+
s.n AS repo_sessions, a.wasted_tokens
|
|
90
|
+
FROM agg a
|
|
91
|
+
JOIN (SELECT repo, count(*) AS n FROM sessions GROUP BY repo) s USING (repo)
|
|
92
|
+
WHERE a.sessions >= 3 AND a.sessions >= 0.25 * s.n
|
|
93
|
+
ORDER BY a.wasted_tokens DESC, a.sessions DESC
|
|
94
|
+
"""
|
|
95
|
+
if limit is not None:
|
|
96
|
+
query += f" LIMIT {int(limit)}"
|
|
97
|
+
return [
|
|
98
|
+
Finding(
|
|
99
|
+
kind="hotspot",
|
|
100
|
+
repo=repo,
|
|
101
|
+
title=target,
|
|
102
|
+
sessions=sessions,
|
|
103
|
+
tokens=int(wasted or 0),
|
|
104
|
+
detail=f"read-only in {read_only}/{repo_sessions} sessions ({read_only / repo_sessions:.0%})",
|
|
105
|
+
)
|
|
106
|
+
for repo, target, sessions, read_only, repo_sessions, wasted in con.execute(query).fetchall()
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def recovery_loops(con: duckdb.DuckDBPyConnection, limit: int | None = None) -> list[Finding]:
|
|
111
|
+
query = """
|
|
112
|
+
SELECT a.repo, a.target AS failed, a.error_signature, b.target AS recovery,
|
|
113
|
+
count(DISTINCT a.session_id) AS sessions,
|
|
114
|
+
count(*) AS occurrences,
|
|
115
|
+
sum(a.approx_out_tokens) AS wasted_tokens
|
|
116
|
+
FROM tool_events a
|
|
117
|
+
JOIN tool_events b
|
|
118
|
+
ON a.session_id = b.session_id
|
|
119
|
+
AND b.step_idx > a.step_idx
|
|
120
|
+
AND b.step_idx - a.step_idx <= 6
|
|
121
|
+
WHERE a.is_error AND NOT b.is_error
|
|
122
|
+
AND a.action_type = 'exec' AND b.action_type = 'exec'
|
|
123
|
+
AND a.target <> '' AND b.target <> ''
|
|
124
|
+
GROUP BY 1, 2, 3, 4
|
|
125
|
+
HAVING count(DISTINCT a.session_id) >= 2
|
|
126
|
+
ORDER BY sessions DESC, wasted_tokens DESC
|
|
127
|
+
"""
|
|
128
|
+
if limit is not None:
|
|
129
|
+
query += f" LIMIT {int(limit)}"
|
|
130
|
+
return [
|
|
131
|
+
Finding(
|
|
132
|
+
kind="recovery",
|
|
133
|
+
repo=repo,
|
|
134
|
+
title=f"{failed} -> {recovery}",
|
|
135
|
+
sessions=sessions,
|
|
136
|
+
tokens=int(wasted or 0),
|
|
137
|
+
detail=f"{occurrences} recurrences across {sessions} sessions; signature {signature}",
|
|
138
|
+
)
|
|
139
|
+
for repo, failed, signature, recovery, sessions, occurrences, wasted in con.execute(query).fetchall()
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def context_blasts(con: duckdb.DuckDBPyConnection, limit: int | None = None) -> list[Finding]:
|
|
144
|
+
query = """
|
|
145
|
+
SELECT repo, target, tool_name, action_type,
|
|
146
|
+
count(DISTINCT session_id) AS sessions,
|
|
147
|
+
count(*) AS calls,
|
|
148
|
+
avg(approx_out_tokens) AS avg_tokens,
|
|
149
|
+
sum(approx_out_tokens) AS total_tokens
|
|
150
|
+
FROM tool_events
|
|
151
|
+
WHERE action_type IN ('exec', 'read')
|
|
152
|
+
AND target <> ''
|
|
153
|
+
AND target NOT LIKE '<%'
|
|
154
|
+
AND target NOT LIKE '%>%'
|
|
155
|
+
AND target NOT LIKE '%;'
|
|
156
|
+
AND target NOT LIKE 'skill:%'
|
|
157
|
+
AND target NOT LIKE 'xd:%'
|
|
158
|
+
GROUP BY 1, 2, 3, 4
|
|
159
|
+
HAVING avg(approx_out_tokens) >= 1500 AND count(DISTINCT session_id) >= 2
|
|
160
|
+
AND count(*) >= 2
|
|
161
|
+
ORDER BY total_tokens DESC
|
|
162
|
+
"""
|
|
163
|
+
if limit is not None:
|
|
164
|
+
query += f" LIMIT {int(limit)}"
|
|
165
|
+
return [
|
|
166
|
+
Finding(
|
|
167
|
+
kind="context",
|
|
168
|
+
repo=repo,
|
|
169
|
+
title=target,
|
|
170
|
+
sessions=sessions,
|
|
171
|
+
tokens=int(total or 0),
|
|
172
|
+
detail=f"{action_type} via {tool_name}: avg {avg:.0f} tokens over {calls} calls",
|
|
173
|
+
)
|
|
174
|
+
for repo, target, tool_name, action_type, sessions, calls, avg, total in con.execute(query).fetchall()
|
|
175
|
+
]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def all_findings(con: duckdb.DuckDBPyConnection, limit: int = 25) -> list[Finding]:
|
|
179
|
+
findings: list[Finding] = []
|
|
180
|
+
findings.extend(hotspots(con, limit=limit))
|
|
181
|
+
findings.extend(recovery_loops(con, limit=limit))
|
|
182
|
+
findings.extend(context_blasts(con, limit=limit))
|
|
183
|
+
return findings
|