trace2rules 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trace2rules/__init__.py +1 -0
- trace2rules/cli.py +543 -0
- trace2rules/cluster.py +348 -0
- trace2rules/db.py +84 -0
- trace2rules/detectors.py +183 -0
- trace2rules/ingest.py +717 -0
- trace2rules/labels.py +431 -0
- trace2rules/models.py +33 -0
- trace2rules/patterns.py +159 -0
- trace2rules/sources/__init__.py +1 -0
- trace2rules/sources/agent_trajectory.py +456 -0
- trace2rules/sources/common.py +99 -0
- trace2rules/sources/opencode.py +154 -0
- trace2rules/sources/swe_chat.py +117 -0
- trace2rules/synthesize.py +538 -0
- trace2rules-0.1.0.dist-info/METADATA +323 -0
- trace2rules-0.1.0.dist-info/RECORD +20 -0
- trace2rules-0.1.0.dist-info/WHEEL +4 -0
- trace2rules-0.1.0.dist-info/entry_points.txt +3 -0
- trace2rules-0.1.0.dist-info/licenses/LICENSE +21 -0
trace2rules/labels.py
ADDED
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import duckdb
|
|
15
|
+
|
|
16
|
+
from .db import DB_PATH, connect
|
|
17
|
+
|
|
18
|
+
DEFAULT_MODEL = "clef-flash"
|
|
19
|
+
DEFAULT_RPM = 270
|
|
20
|
+
DEFAULT_WORKERS = 6
|
|
21
|
+
|
|
22
|
+
INTENT_CRITERIA = {
|
|
23
|
+
"explore": "List or inspect the workspace, repo root, or directory structure",
|
|
24
|
+
"read": "Read a specific file",
|
|
25
|
+
"search": "Search for a symbol, pattern, or filename",
|
|
26
|
+
"edit": "Create or modify a file",
|
|
27
|
+
"run_tests": "Run pytest or a specific test file/selector",
|
|
28
|
+
"run_script": "Execute a reproduction, verification, or ad-hoc script",
|
|
29
|
+
"install": "Install the project, dependencies, or a package",
|
|
30
|
+
"env_check": "Probe imports, versions, paths, or environment variables",
|
|
31
|
+
"git": "Inspect or modify git state",
|
|
32
|
+
"other": "None of the above",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
OBJECT_CRITERIA = {
|
|
36
|
+
"file": "A source, test, or config file",
|
|
37
|
+
"directory": "A directory, package root, or workspace",
|
|
38
|
+
"test_scope": "A test file, test selector, or suite",
|
|
39
|
+
"package": "An installed or local package",
|
|
40
|
+
"module": "An importable Python module",
|
|
41
|
+
"config": "Project configuration",
|
|
42
|
+
"unknown": "Cannot determine",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
QUESTIONS = {
|
|
46
|
+
"intent": {
|
|
47
|
+
"type": "choice",
|
|
48
|
+
"instructions": "What is the agent trying to accomplish?",
|
|
49
|
+
"criteria": INTENT_CRITERIA,
|
|
50
|
+
},
|
|
51
|
+
"object_type": {
|
|
52
|
+
"type": "choice",
|
|
53
|
+
"instructions": "What kind of object is acted on?",
|
|
54
|
+
"criteria": OBJECT_CRITERIA,
|
|
55
|
+
},
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
MODELS: dict[str, dict[str, str]] = {
|
|
59
|
+
"clef-flash": {
|
|
60
|
+
"name": "clef-flash",
|
|
61
|
+
"provider": "cloudflare",
|
|
62
|
+
"model_id": "@cf/cloudflare/clef-flash",
|
|
63
|
+
"wire_model": "clef-flash",
|
|
64
|
+
"style": "clef",
|
|
65
|
+
},
|
|
66
|
+
"clef": {
|
|
67
|
+
"name": "clef",
|
|
68
|
+
"provider": "cloudflare",
|
|
69
|
+
"model_id": "@cf/cloudflare/clef",
|
|
70
|
+
"wire_model": "clef",
|
|
71
|
+
"style": "clef",
|
|
72
|
+
},
|
|
73
|
+
"jev": {
|
|
74
|
+
"name": "jev",
|
|
75
|
+
"provider": "cloudflare",
|
|
76
|
+
"model_id": "typesafe/jev",
|
|
77
|
+
"wire_model": "typesafe/jev",
|
|
78
|
+
"style": "jev",
|
|
79
|
+
},
|
|
80
|
+
"gemini-decision-1": {
|
|
81
|
+
"name": "gemini-decision-1",
|
|
82
|
+
"provider": "gemini",
|
|
83
|
+
"model_id": "gemini-decision-1",
|
|
84
|
+
},
|
|
85
|
+
"gpt-6-luna": {
|
|
86
|
+
"name": "gpt-6-luna",
|
|
87
|
+
"provider": "openai",
|
|
88
|
+
"model_id": "gpt-6-luna",
|
|
89
|
+
},
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
MODEL_ALIASES = {
|
|
93
|
+
"luna": "gpt-6-luna",
|
|
94
|
+
"gemini": "gemini-decision-1",
|
|
95
|
+
"gemini-decision": "gemini-decision-1",
|
|
96
|
+
"jev-1.13": "jev",
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class RateLimiter:
|
|
101
|
+
def __init__(self, rpm: int) -> None:
|
|
102
|
+
self.interval = 60.0 / max(rpm, 1)
|
|
103
|
+
self.lock = threading.Lock()
|
|
104
|
+
self.next_slot = time.monotonic()
|
|
105
|
+
|
|
106
|
+
def wait(self) -> None:
|
|
107
|
+
with self.lock:
|
|
108
|
+
now = time.monotonic()
|
|
109
|
+
if self.next_slot > now:
|
|
110
|
+
time.sleep(self.next_slot - now)
|
|
111
|
+
now = time.monotonic()
|
|
112
|
+
self.next_slot = max(now, self.next_slot) + self.interval
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _wrangler_ready() -> bool:
|
|
116
|
+
if not shutil.which("wrangler"):
|
|
117
|
+
return False
|
|
118
|
+
try:
|
|
119
|
+
result = subprocess.run(
|
|
120
|
+
["wrangler", "whoami", "--json"],
|
|
121
|
+
capture_output=True,
|
|
122
|
+
text=True,
|
|
123
|
+
timeout=5,
|
|
124
|
+
check=False,
|
|
125
|
+
)
|
|
126
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
127
|
+
return False
|
|
128
|
+
return result.returncode == 0 and '"loggedIn": true' in result.stdout
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def detect_model() -> str | None:
|
|
132
|
+
"""Pick the cheapest available decision backend without requiring config."""
|
|
133
|
+
configured = os.getenv("TRACE2RULES_DECISION_MODEL")
|
|
134
|
+
if configured:
|
|
135
|
+
return configured
|
|
136
|
+
if _wrangler_ready():
|
|
137
|
+
return "clef-flash"
|
|
138
|
+
if os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY"):
|
|
139
|
+
return "gemini-decision-1"
|
|
140
|
+
if os.getenv("OPENAI_API_KEY"):
|
|
141
|
+
return "gpt-6-luna"
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _resolve_model(name: str | None) -> dict[str, str]:
|
|
146
|
+
selected = (name or os.getenv("TRACE2RULES_DECISION_MODEL") or detect_model() or "").strip()
|
|
147
|
+
if not selected:
|
|
148
|
+
raise RuntimeError(
|
|
149
|
+
"No decision backend found; set GEMINI_API_KEY, OPENAI_API_KEY, "
|
|
150
|
+
"TRACE2RULES_DECISION_MODEL, or install wrangler."
|
|
151
|
+
)
|
|
152
|
+
selected = MODEL_ALIASES.get(selected, selected)
|
|
153
|
+
if selected not in MODELS:
|
|
154
|
+
raise RuntimeError(
|
|
155
|
+
"Unknown decision model "
|
|
156
|
+
f"{selected!r}; choose one of: {', '.join(sorted(MODELS))}."
|
|
157
|
+
)
|
|
158
|
+
return MODELS[selected]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _wrangler_json(*args: str) -> dict:
|
|
162
|
+
try:
|
|
163
|
+
output = subprocess.check_output(
|
|
164
|
+
["wrangler", *args], text=True, stderr=subprocess.PIPE
|
|
165
|
+
)
|
|
166
|
+
except FileNotFoundError as error:
|
|
167
|
+
raise RuntimeError(
|
|
168
|
+
"wrangler not found; install it with `npm install -g wrangler`."
|
|
169
|
+
) from error
|
|
170
|
+
except subprocess.CalledProcessError as error:
|
|
171
|
+
detail = (error.stderr or "").strip()[:300]
|
|
172
|
+
raise RuntimeError(
|
|
173
|
+
"wrangler is not authenticated; run `wrangler login`."
|
|
174
|
+
+ (f" ({detail})" if detail else "")
|
|
175
|
+
) from error
|
|
176
|
+
return json.loads(output)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _cloudflare_auth() -> tuple[str, str]:
|
|
180
|
+
token = _wrangler_json("auth", "token", "--json")["token"]
|
|
181
|
+
whoami = _wrangler_json("whoami", "--json")
|
|
182
|
+
accounts = whoami.get("accounts") or []
|
|
183
|
+
if not accounts:
|
|
184
|
+
raise RuntimeError("wrangler is logged in but exposes no account")
|
|
185
|
+
return accounts[0]["id"], token
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _auth_for(spec: dict[str, str]):
|
|
189
|
+
provider = spec["provider"]
|
|
190
|
+
if provider == "cloudflare":
|
|
191
|
+
return _cloudflare_auth()
|
|
192
|
+
if provider == "gemini":
|
|
193
|
+
key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
|
|
194
|
+
if not key:
|
|
195
|
+
raise RuntimeError(
|
|
196
|
+
"gemini-decision-1 requires GEMINI_API_KEY or GOOGLE_API_KEY."
|
|
197
|
+
)
|
|
198
|
+
return key
|
|
199
|
+
if provider == "openai":
|
|
200
|
+
key = os.getenv("OPENAI_API_KEY")
|
|
201
|
+
if not key:
|
|
202
|
+
raise RuntimeError("gpt-6-luna requires OPENAI_API_KEY.")
|
|
203
|
+
return key
|
|
204
|
+
raise RuntimeError(f"Unsupported decision provider: {provider}")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _openai_questions() -> list[dict]:
|
|
208
|
+
return [
|
|
209
|
+
{
|
|
210
|
+
"type": "choice",
|
|
211
|
+
"name": name,
|
|
212
|
+
"instructions": question["instructions"],
|
|
213
|
+
"choices": [
|
|
214
|
+
{"value": value, "description": description}
|
|
215
|
+
for value, description in question["criteria"].items()
|
|
216
|
+
],
|
|
217
|
+
}
|
|
218
|
+
for name, question in QUESTIONS.items()
|
|
219
|
+
]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _openai_answers(payload: dict) -> dict:
|
|
223
|
+
answers = payload.get("answers") or []
|
|
224
|
+
return {
|
|
225
|
+
answer.get("name"): {
|
|
226
|
+
"choice": answer.get("choice"),
|
|
227
|
+
"confidence": answer.get("confidence") or 0.0,
|
|
228
|
+
}
|
|
229
|
+
for answer in answers
|
|
230
|
+
if answer.get("name")
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _post(auth, state: dict, spec: dict[str, str]) -> dict:
|
|
235
|
+
provider = spec["provider"]
|
|
236
|
+
if provider == "cloudflare":
|
|
237
|
+
account, token = auth
|
|
238
|
+
if spec["style"] == "jev":
|
|
239
|
+
url = "https://api.cloudflare.com/client/v4/accounts/{}/ai/run".format(account)
|
|
240
|
+
body = {
|
|
241
|
+
"model": spec["wire_model"],
|
|
242
|
+
"input": {"state": state, "questions": QUESTIONS},
|
|
243
|
+
}
|
|
244
|
+
else:
|
|
245
|
+
url = (
|
|
246
|
+
"https://api.cloudflare.com/client/v4/accounts/{}/ai/run/{}".format(
|
|
247
|
+
account, spec["model_id"]
|
|
248
|
+
)
|
|
249
|
+
)
|
|
250
|
+
body = {
|
|
251
|
+
"model": spec["wire_model"],
|
|
252
|
+
"state": state,
|
|
253
|
+
"questions": QUESTIONS,
|
|
254
|
+
}
|
|
255
|
+
request = urllib.request.Request(
|
|
256
|
+
url,
|
|
257
|
+
data=json.dumps(body).encode(),
|
|
258
|
+
headers={
|
|
259
|
+
"Authorization": f"Bearer {token}",
|
|
260
|
+
"Content-Type": "application/json",
|
|
261
|
+
},
|
|
262
|
+
method="POST",
|
|
263
|
+
)
|
|
264
|
+
with urllib.request.urlopen(request, timeout=90) as response:
|
|
265
|
+
payload = json.load(response)
|
|
266
|
+
if not payload.get("success"):
|
|
267
|
+
raise RuntimeError(f"{spec['name']} error: {payload.get('errors')}")
|
|
268
|
+
return payload["result"]["answers"]
|
|
269
|
+
|
|
270
|
+
if provider == "gemini":
|
|
271
|
+
body = {
|
|
272
|
+
"model": spec["model_id"],
|
|
273
|
+
"input": json.dumps(state, ensure_ascii=False),
|
|
274
|
+
"questions": QUESTIONS,
|
|
275
|
+
}
|
|
276
|
+
request = urllib.request.Request(
|
|
277
|
+
"https://generativelanguage.googleapis.com/v1beta/decisions",
|
|
278
|
+
data=json.dumps(body).encode(),
|
|
279
|
+
headers={
|
|
280
|
+
"x-goog-api-key": auth,
|
|
281
|
+
"Content-Type": "application/json",
|
|
282
|
+
},
|
|
283
|
+
method="POST",
|
|
284
|
+
)
|
|
285
|
+
with urllib.request.urlopen(request, timeout=90) as response:
|
|
286
|
+
payload = json.load(response)
|
|
287
|
+
return payload["answers"]
|
|
288
|
+
|
|
289
|
+
if provider == "openai":
|
|
290
|
+
body = {
|
|
291
|
+
"model": spec["model_id"],
|
|
292
|
+
"input": json.dumps(state, ensure_ascii=False),
|
|
293
|
+
"questions": _openai_questions(),
|
|
294
|
+
}
|
|
295
|
+
request = urllib.request.Request(
|
|
296
|
+
"https://api.openai.com/v1/decisions",
|
|
297
|
+
data=json.dumps(body).encode(),
|
|
298
|
+
headers={
|
|
299
|
+
"Authorization": f"Bearer {auth}",
|
|
300
|
+
"Content-Type": "application/json",
|
|
301
|
+
},
|
|
302
|
+
method="POST",
|
|
303
|
+
)
|
|
304
|
+
with urllib.request.urlopen(request, timeout=90) as response:
|
|
305
|
+
payload = json.load(response)
|
|
306
|
+
return _openai_answers(payload)
|
|
307
|
+
|
|
308
|
+
raise RuntimeError(f"Unsupported decision provider: {provider}")
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _retry(auth, state: dict, limiter: RateLimiter, spec: dict[str, str]) -> dict:
|
|
312
|
+
last_error: Exception | None = None
|
|
313
|
+
for attempt in range(6):
|
|
314
|
+
limiter.wait()
|
|
315
|
+
try:
|
|
316
|
+
return _post(auth, state, spec)
|
|
317
|
+
except urllib.error.HTTPError as error:
|
|
318
|
+
detail = error.read().decode(errors="replace")[:500]
|
|
319
|
+
if error.code == 429 or error.code >= 500:
|
|
320
|
+
retry_after = error.headers.get("Retry-After")
|
|
321
|
+
delay = float(retry_after) if retry_after else 2**attempt
|
|
322
|
+
time.sleep(min(delay, 60))
|
|
323
|
+
last_error = RuntimeError(f"HTTP {error.code}: {detail}")
|
|
324
|
+
continue
|
|
325
|
+
raise RuntimeError(f"HTTP {error.code}: {detail}") from error
|
|
326
|
+
except (urllib.error.URLError, TimeoutError) as error:
|
|
327
|
+
last_error = error
|
|
328
|
+
time.sleep(2**attempt)
|
|
329
|
+
raise RuntimeError(
|
|
330
|
+
f"{spec['name']} failed after retries: {last_error}"
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _pending(
|
|
335
|
+
con: duckdb.DuckDBPyConnection, limit: int | None
|
|
336
|
+
) -> list[tuple[str, str, str, str]]:
|
|
337
|
+
query = """
|
|
338
|
+
SELECT e.tool_name, e.action_type, e.target, min(e.raw_target) AS raw_example
|
|
339
|
+
FROM tool_events e
|
|
340
|
+
LEFT JOIN target_labels l
|
|
341
|
+
ON e.tool_name = l.tool_name
|
|
342
|
+
AND e.action_type = l.action_type
|
|
343
|
+
AND e.target = l.target
|
|
344
|
+
WHERE l.tool_name IS NULL
|
|
345
|
+
GROUP BY 1, 2, 3
|
|
346
|
+
ORDER BY count(*) DESC
|
|
347
|
+
"""
|
|
348
|
+
if limit is not None:
|
|
349
|
+
query += f" LIMIT {int(limit)}"
|
|
350
|
+
return con.execute(query).fetchall()
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _state(row: tuple[str, str, str, str]) -> dict:
|
|
354
|
+
tool_name, action_type, target, raw_example = row
|
|
355
|
+
return {
|
|
356
|
+
"tool": tool_name,
|
|
357
|
+
"action_type": action_type,
|
|
358
|
+
"normalized_target": target[:500],
|
|
359
|
+
"raw_example": (raw_example or "").replace("\x00", " ")[:500],
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def label_targets(
|
|
364
|
+
db_path: str | Path = DB_PATH,
|
|
365
|
+
limit: int | None = None,
|
|
366
|
+
rpm: int = DEFAULT_RPM,
|
|
367
|
+
workers: int = DEFAULT_WORKERS,
|
|
368
|
+
model: str | None = None,
|
|
369
|
+
) -> tuple[int, int]:
|
|
370
|
+
con = connect(db_path)
|
|
371
|
+
rows = _pending(con, limit)
|
|
372
|
+
if not rows:
|
|
373
|
+
con.close()
|
|
374
|
+
return 0, 0
|
|
375
|
+
|
|
376
|
+
spec = _resolve_model(model)
|
|
377
|
+
auth = _auth_for(spec)
|
|
378
|
+
limiter = RateLimiter(rpm)
|
|
379
|
+
done = 0
|
|
380
|
+
failed = 0
|
|
381
|
+
batch: list[tuple] = []
|
|
382
|
+
started = time.monotonic()
|
|
383
|
+
|
|
384
|
+
def work(row: tuple[str, str, str, str]) -> tuple:
|
|
385
|
+
result = _retry(auth, _state(row), limiter, spec)
|
|
386
|
+
intent = result["intent"]
|
|
387
|
+
object_type = result["object_type"]
|
|
388
|
+
return (
|
|
389
|
+
row[0],
|
|
390
|
+
row[1],
|
|
391
|
+
row[2],
|
|
392
|
+
intent["choice"],
|
|
393
|
+
float(intent.get("confidence") or 0.0),
|
|
394
|
+
object_type["choice"],
|
|
395
|
+
float(object_type.get("confidence") or 0.0),
|
|
396
|
+
spec["name"],
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
try:
|
|
400
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
401
|
+
futures = {pool.submit(work, row): row for row in rows}
|
|
402
|
+
for future in as_completed(futures):
|
|
403
|
+
row = futures[future]
|
|
404
|
+
try:
|
|
405
|
+
batch.append(future.result())
|
|
406
|
+
done += 1
|
|
407
|
+
except Exception as error: # noqa: BLE001 - report per-row failures
|
|
408
|
+
failed += 1
|
|
409
|
+
print(f"FAILED {row[0]} {row[2][:80]}: {error}")
|
|
410
|
+
if len(batch) >= 100:
|
|
411
|
+
con.executemany(
|
|
412
|
+
"INSERT OR REPLACE INTO target_labels VALUES (?, ?, ?, ?, ?, ?, ?, ?, current_timestamp)",
|
|
413
|
+
batch,
|
|
414
|
+
)
|
|
415
|
+
batch.clear()
|
|
416
|
+
if (done + failed) % 100 == 0:
|
|
417
|
+
elapsed = time.monotonic() - started
|
|
418
|
+
rate = (done + failed) / elapsed * 60 if elapsed else 0
|
|
419
|
+
print(
|
|
420
|
+
f"{done + failed}/{len(rows)} ok={done} failed={failed} "
|
|
421
|
+
f"{rate:.0f}/min",
|
|
422
|
+
flush=True,
|
|
423
|
+
)
|
|
424
|
+
if batch:
|
|
425
|
+
con.executemany(
|
|
426
|
+
"INSERT OR REPLACE INTO target_labels VALUES (?, ?, ?, ?, ?, ?, ?, ?, current_timestamp)",
|
|
427
|
+
batch,
|
|
428
|
+
)
|
|
429
|
+
finally:
|
|
430
|
+
con.close()
|
|
431
|
+
return done, failed
|
trace2rules/models.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(slots=True)
|
|
7
|
+
class ToolCall:
|
|
8
|
+
"""A normalized tool call before it is classified for DuckDB."""
|
|
9
|
+
|
|
10
|
+
tool_name: str
|
|
11
|
+
raw_target: str
|
|
12
|
+
observation: str = ""
|
|
13
|
+
action_type: str | None = None
|
|
14
|
+
target: str | None = None
|
|
15
|
+
verb: str | None = None
|
|
16
|
+
object_id: str | None = None
|
|
17
|
+
exit_code: int | None = None
|
|
18
|
+
is_error: bool | None = None
|
|
19
|
+
segment_idx: int = 0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class Session:
|
|
24
|
+
"""A normalized session accepted by the shared DuckDB writer."""
|
|
25
|
+
|
|
26
|
+
session_id: str
|
|
27
|
+
repo: str
|
|
28
|
+
instance_id: str = ""
|
|
29
|
+
resolved: bool = False
|
|
30
|
+
calls: list[ToolCall] = field(default_factory=list)
|
|
31
|
+
source: str = ""
|
|
32
|
+
agent: str = ""
|
|
33
|
+
success_score: float | None = None
|
trace2rules/patterns.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
import duckdb
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class SequencePattern:
|
|
11
|
+
sequence: str
|
|
12
|
+
sessions: int
|
|
13
|
+
occurrences: int
|
|
14
|
+
tokens: int
|
|
15
|
+
repos: tuple[str, ...]
|
|
16
|
+
example: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _token(
|
|
20
|
+
action: str,
|
|
21
|
+
object_id: str,
|
|
22
|
+
outcome: str,
|
|
23
|
+
intent: str,
|
|
24
|
+
object_type: str,
|
|
25
|
+
intent_confidence: float,
|
|
26
|
+
object_confidence: float,
|
|
27
|
+
min_intent: float,
|
|
28
|
+
min_object: float,
|
|
29
|
+
) -> str:
|
|
30
|
+
obj = object_id or "?"
|
|
31
|
+
obj = " ".join(obj.split())
|
|
32
|
+
if len(obj) > 140:
|
|
33
|
+
obj = obj[:137] + "..."
|
|
34
|
+
if (
|
|
35
|
+
intent
|
|
36
|
+
and intent != "other"
|
|
37
|
+
and object_type
|
|
38
|
+
and object_type != "unknown"
|
|
39
|
+
and intent_confidence >= min_intent
|
|
40
|
+
and object_confidence >= min_object
|
|
41
|
+
):
|
|
42
|
+
return f"{intent}/{object_type}:{obj}:{outcome}"
|
|
43
|
+
return f"{action}:{obj}:{outcome}"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def mine_sequences(
|
|
47
|
+
con: duckdb.DuckDBPyConnection,
|
|
48
|
+
min_sessions: int = 3,
|
|
49
|
+
max_len: int = 4,
|
|
50
|
+
top: int = 25,
|
|
51
|
+
min_confidence: float = 0.4,
|
|
52
|
+
) -> list[SequencePattern]:
|
|
53
|
+
rows = con.execute(
|
|
54
|
+
"""
|
|
55
|
+
SELECT e.session_id, e.step_idx, e.segment_idx, e.repo,
|
|
56
|
+
e.action_type, e.object_id, e.outcome, e.approx_out_tokens,
|
|
57
|
+
e.raw_target, l.intent, l.object_type,
|
|
58
|
+
l.intent_confidence, l.object_confidence
|
|
59
|
+
FROM tool_events e
|
|
60
|
+
LEFT JOIN target_labels l
|
|
61
|
+
ON e.tool_name = l.tool_name
|
|
62
|
+
AND e.action_type = l.action_type
|
|
63
|
+
AND e.target = l.target
|
|
64
|
+
WHERE e.tool_name NOT IN ('think', 'finish', 'task_tracker')
|
|
65
|
+
AND e.action_type <> 'other'
|
|
66
|
+
ORDER BY e.session_id, e.step_idx
|
|
67
|
+
"""
|
|
68
|
+
).fetchall()
|
|
69
|
+
|
|
70
|
+
sessions: dict[str, list[tuple]] = defaultdict(list)
|
|
71
|
+
for row in rows:
|
|
72
|
+
sessions[row[0]].append(row)
|
|
73
|
+
|
|
74
|
+
aggregates: dict[tuple[str, ...], dict] = {}
|
|
75
|
+
object_floor = max(min_confidence, 0.6)
|
|
76
|
+
for session_id, events in sessions.items():
|
|
77
|
+
tokens = [
|
|
78
|
+
_token(
|
|
79
|
+
event[4],
|
|
80
|
+
event[5]
|
|
81
|
+
if event[5] not in {"", "<path>", "unknown", "?"}
|
|
82
|
+
else (event[8] or "?"),
|
|
83
|
+
event[6],
|
|
84
|
+
event[9] or "",
|
|
85
|
+
event[10] or "",
|
|
86
|
+
float(event[11] or 0.0),
|
|
87
|
+
float(event[12] or 0.0),
|
|
88
|
+
min_confidence,
|
|
89
|
+
object_floor,
|
|
90
|
+
)
|
|
91
|
+
for event in events
|
|
92
|
+
]
|
|
93
|
+
for length in range(2, max_len + 1):
|
|
94
|
+
for start in range(len(events) - length + 1):
|
|
95
|
+
window = events[start : start + length]
|
|
96
|
+
if any(
|
|
97
|
+
window[index][1] + 1 != window[index + 1][1]
|
|
98
|
+
or window[index][2] != window[index + 1][2]
|
|
99
|
+
for index in range(len(window) - 1)
|
|
100
|
+
):
|
|
101
|
+
continue
|
|
102
|
+
window_tokens = tuple(tokens[start : start + length])
|
|
103
|
+
if not any(event[6] == "err" for event in window):
|
|
104
|
+
continue
|
|
105
|
+
agg = aggregates.setdefault(
|
|
106
|
+
window_tokens,
|
|
107
|
+
{
|
|
108
|
+
"sessions": set(),
|
|
109
|
+
"occurrences": 0,
|
|
110
|
+
"tokens": 0,
|
|
111
|
+
"repos": set(),
|
|
112
|
+
"example": "",
|
|
113
|
+
},
|
|
114
|
+
)
|
|
115
|
+
agg["sessions"].add(session_id)
|
|
116
|
+
agg["repos"].update(event[3] for event in window)
|
|
117
|
+
agg["occurrences"] += 1
|
|
118
|
+
agg["tokens"] += sum(
|
|
119
|
+
event[7] for event in window if event[6] == "err"
|
|
120
|
+
)
|
|
121
|
+
if not agg["example"]:
|
|
122
|
+
agg["example"] = " -> ".join(
|
|
123
|
+
(event[8] or event[5] or "").replace("\x00", " ")
|
|
124
|
+
for event in window
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
candidates = [
|
|
128
|
+
SequencePattern(
|
|
129
|
+
sequence=" -> ".join(tokens),
|
|
130
|
+
sessions=len(value["sessions"]),
|
|
131
|
+
occurrences=value["occurrences"],
|
|
132
|
+
tokens=value["tokens"],
|
|
133
|
+
repos=tuple(sorted(value["repos"])),
|
|
134
|
+
example=value["example"],
|
|
135
|
+
)
|
|
136
|
+
for tokens, value in aggregates.items()
|
|
137
|
+
if len(value["sessions"]) >= min_sessions
|
|
138
|
+
]
|
|
139
|
+
|
|
140
|
+
# Prefer maximal windows; a shorter prefix with the same support is redundant.
|
|
141
|
+
candidates.sort(key=lambda pattern: len(pattern.sequence.split(" -> ")), reverse=True)
|
|
142
|
+
kept: list[SequencePattern] = []
|
|
143
|
+
for pattern in candidates:
|
|
144
|
+
if any(
|
|
145
|
+
pattern.sequence in longer.sequence
|
|
146
|
+
and pattern.sessions <= longer.sessions
|
|
147
|
+
for longer in kept
|
|
148
|
+
):
|
|
149
|
+
continue
|
|
150
|
+
object_ids = {
|
|
151
|
+
token.rsplit(":", 2)[1]
|
|
152
|
+
for token in pattern.sequence.split(" -> ")
|
|
153
|
+
}
|
|
154
|
+
if len(object_ids) == 1 and pattern.sessions < 5:
|
|
155
|
+
continue
|
|
156
|
+
kept.append(pattern)
|
|
157
|
+
|
|
158
|
+
kept.sort(key=lambda pattern: (pattern.sessions, pattern.tokens), reverse=True)
|
|
159
|
+
return kept[:top]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Session sources that produce canonical trace2rules records."""
|