trace2rules 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
trace2rules/labels.py ADDED
@@ -0,0 +1,431 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import shutil
6
+ import subprocess
7
+ import threading
8
+ import time
9
+ import urllib.error
10
+ import urllib.request
11
+ from concurrent.futures import ThreadPoolExecutor, as_completed
12
+ from pathlib import Path
13
+
14
+ import duckdb
15
+
16
+ from .db import DB_PATH, connect
17
+
18
+ DEFAULT_MODEL = "clef-flash"
19
+ DEFAULT_RPM = 270
20
+ DEFAULT_WORKERS = 6
21
+
22
+ INTENT_CRITERIA = {
23
+ "explore": "List or inspect the workspace, repo root, or directory structure",
24
+ "read": "Read a specific file",
25
+ "search": "Search for a symbol, pattern, or filename",
26
+ "edit": "Create or modify a file",
27
+ "run_tests": "Run pytest or a specific test file/selector",
28
+ "run_script": "Execute a reproduction, verification, or ad-hoc script",
29
+ "install": "Install the project, dependencies, or a package",
30
+ "env_check": "Probe imports, versions, paths, or environment variables",
31
+ "git": "Inspect or modify git state",
32
+ "other": "None of the above",
33
+ }
34
+
35
+ OBJECT_CRITERIA = {
36
+ "file": "A source, test, or config file",
37
+ "directory": "A directory, package root, or workspace",
38
+ "test_scope": "A test file, test selector, or suite",
39
+ "package": "An installed or local package",
40
+ "module": "An importable Python module",
41
+ "config": "Project configuration",
42
+ "unknown": "Cannot determine",
43
+ }
44
+
45
+ QUESTIONS = {
46
+ "intent": {
47
+ "type": "choice",
48
+ "instructions": "What is the agent trying to accomplish?",
49
+ "criteria": INTENT_CRITERIA,
50
+ },
51
+ "object_type": {
52
+ "type": "choice",
53
+ "instructions": "What kind of object is acted on?",
54
+ "criteria": OBJECT_CRITERIA,
55
+ },
56
+ }
57
+
58
+ MODELS: dict[str, dict[str, str]] = {
59
+ "clef-flash": {
60
+ "name": "clef-flash",
61
+ "provider": "cloudflare",
62
+ "model_id": "@cf/cloudflare/clef-flash",
63
+ "wire_model": "clef-flash",
64
+ "style": "clef",
65
+ },
66
+ "clef": {
67
+ "name": "clef",
68
+ "provider": "cloudflare",
69
+ "model_id": "@cf/cloudflare/clef",
70
+ "wire_model": "clef",
71
+ "style": "clef",
72
+ },
73
+ "jev": {
74
+ "name": "jev",
75
+ "provider": "cloudflare",
76
+ "model_id": "typesafe/jev",
77
+ "wire_model": "typesafe/jev",
78
+ "style": "jev",
79
+ },
80
+ "gemini-decision-1": {
81
+ "name": "gemini-decision-1",
82
+ "provider": "gemini",
83
+ "model_id": "gemini-decision-1",
84
+ },
85
+ "gpt-6-luna": {
86
+ "name": "gpt-6-luna",
87
+ "provider": "openai",
88
+ "model_id": "gpt-6-luna",
89
+ },
90
+ }
91
+
92
+ MODEL_ALIASES = {
93
+ "luna": "gpt-6-luna",
94
+ "gemini": "gemini-decision-1",
95
+ "gemini-decision": "gemini-decision-1",
96
+ "jev-1.13": "jev",
97
+ }
98
+
99
+
100
+ class RateLimiter:
101
+ def __init__(self, rpm: int) -> None:
102
+ self.interval = 60.0 / max(rpm, 1)
103
+ self.lock = threading.Lock()
104
+ self.next_slot = time.monotonic()
105
+
106
+ def wait(self) -> None:
107
+ with self.lock:
108
+ now = time.monotonic()
109
+ if self.next_slot > now:
110
+ time.sleep(self.next_slot - now)
111
+ now = time.monotonic()
112
+ self.next_slot = max(now, self.next_slot) + self.interval
113
+
114
+
115
+ def _wrangler_ready() -> bool:
116
+ if not shutil.which("wrangler"):
117
+ return False
118
+ try:
119
+ result = subprocess.run(
120
+ ["wrangler", "whoami", "--json"],
121
+ capture_output=True,
122
+ text=True,
123
+ timeout=5,
124
+ check=False,
125
+ )
126
+ except (OSError, subprocess.TimeoutExpired):
127
+ return False
128
+ return result.returncode == 0 and '"loggedIn": true' in result.stdout
129
+
130
+
131
+ def detect_model() -> str | None:
132
+ """Pick the cheapest available decision backend without requiring config."""
133
+ configured = os.getenv("TRACE2RULES_DECISION_MODEL")
134
+ if configured:
135
+ return configured
136
+ if _wrangler_ready():
137
+ return "clef-flash"
138
+ if os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY"):
139
+ return "gemini-decision-1"
140
+ if os.getenv("OPENAI_API_KEY"):
141
+ return "gpt-6-luna"
142
+ return None
143
+
144
+
145
+ def _resolve_model(name: str | None) -> dict[str, str]:
146
+ selected = (name or os.getenv("TRACE2RULES_DECISION_MODEL") or detect_model() or "").strip()
147
+ if not selected:
148
+ raise RuntimeError(
149
+ "No decision backend found; set GEMINI_API_KEY, OPENAI_API_KEY, "
150
+ "TRACE2RULES_DECISION_MODEL, or install wrangler."
151
+ )
152
+ selected = MODEL_ALIASES.get(selected, selected)
153
+ if selected not in MODELS:
154
+ raise RuntimeError(
155
+ "Unknown decision model "
156
+ f"{selected!r}; choose one of: {', '.join(sorted(MODELS))}."
157
+ )
158
+ return MODELS[selected]
159
+
160
+
161
+ def _wrangler_json(*args: str) -> dict:
162
+ try:
163
+ output = subprocess.check_output(
164
+ ["wrangler", *args], text=True, stderr=subprocess.PIPE
165
+ )
166
+ except FileNotFoundError as error:
167
+ raise RuntimeError(
168
+ "wrangler not found; install it with `npm install -g wrangler`."
169
+ ) from error
170
+ except subprocess.CalledProcessError as error:
171
+ detail = (error.stderr or "").strip()[:300]
172
+ raise RuntimeError(
173
+ "wrangler is not authenticated; run `wrangler login`."
174
+ + (f" ({detail})" if detail else "")
175
+ ) from error
176
+ return json.loads(output)
177
+
178
+
179
+ def _cloudflare_auth() -> tuple[str, str]:
180
+ token = _wrangler_json("auth", "token", "--json")["token"]
181
+ whoami = _wrangler_json("whoami", "--json")
182
+ accounts = whoami.get("accounts") or []
183
+ if not accounts:
184
+ raise RuntimeError("wrangler is logged in but exposes no account")
185
+ return accounts[0]["id"], token
186
+
187
+
188
+ def _auth_for(spec: dict[str, str]):
189
+ provider = spec["provider"]
190
+ if provider == "cloudflare":
191
+ return _cloudflare_auth()
192
+ if provider == "gemini":
193
+ key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
194
+ if not key:
195
+ raise RuntimeError(
196
+ "gemini-decision-1 requires GEMINI_API_KEY or GOOGLE_API_KEY."
197
+ )
198
+ return key
199
+ if provider == "openai":
200
+ key = os.getenv("OPENAI_API_KEY")
201
+ if not key:
202
+ raise RuntimeError("gpt-6-luna requires OPENAI_API_KEY.")
203
+ return key
204
+ raise RuntimeError(f"Unsupported decision provider: {provider}")
205
+
206
+
207
+ def _openai_questions() -> list[dict]:
208
+ return [
209
+ {
210
+ "type": "choice",
211
+ "name": name,
212
+ "instructions": question["instructions"],
213
+ "choices": [
214
+ {"value": value, "description": description}
215
+ for value, description in question["criteria"].items()
216
+ ],
217
+ }
218
+ for name, question in QUESTIONS.items()
219
+ ]
220
+
221
+
222
+ def _openai_answers(payload: dict) -> dict:
223
+ answers = payload.get("answers") or []
224
+ return {
225
+ answer.get("name"): {
226
+ "choice": answer.get("choice"),
227
+ "confidence": answer.get("confidence") or 0.0,
228
+ }
229
+ for answer in answers
230
+ if answer.get("name")
231
+ }
232
+
233
+
234
+ def _post(auth, state: dict, spec: dict[str, str]) -> dict:
235
+ provider = spec["provider"]
236
+ if provider == "cloudflare":
237
+ account, token = auth
238
+ if spec["style"] == "jev":
239
+ url = "https://api.cloudflare.com/client/v4/accounts/{}/ai/run".format(account)
240
+ body = {
241
+ "model": spec["wire_model"],
242
+ "input": {"state": state, "questions": QUESTIONS},
243
+ }
244
+ else:
245
+ url = (
246
+ "https://api.cloudflare.com/client/v4/accounts/{}/ai/run/{}".format(
247
+ account, spec["model_id"]
248
+ )
249
+ )
250
+ body = {
251
+ "model": spec["wire_model"],
252
+ "state": state,
253
+ "questions": QUESTIONS,
254
+ }
255
+ request = urllib.request.Request(
256
+ url,
257
+ data=json.dumps(body).encode(),
258
+ headers={
259
+ "Authorization": f"Bearer {token}",
260
+ "Content-Type": "application/json",
261
+ },
262
+ method="POST",
263
+ )
264
+ with urllib.request.urlopen(request, timeout=90) as response:
265
+ payload = json.load(response)
266
+ if not payload.get("success"):
267
+ raise RuntimeError(f"{spec['name']} error: {payload.get('errors')}")
268
+ return payload["result"]["answers"]
269
+
270
+ if provider == "gemini":
271
+ body = {
272
+ "model": spec["model_id"],
273
+ "input": json.dumps(state, ensure_ascii=False),
274
+ "questions": QUESTIONS,
275
+ }
276
+ request = urllib.request.Request(
277
+ "https://generativelanguage.googleapis.com/v1beta/decisions",
278
+ data=json.dumps(body).encode(),
279
+ headers={
280
+ "x-goog-api-key": auth,
281
+ "Content-Type": "application/json",
282
+ },
283
+ method="POST",
284
+ )
285
+ with urllib.request.urlopen(request, timeout=90) as response:
286
+ payload = json.load(response)
287
+ return payload["answers"]
288
+
289
+ if provider == "openai":
290
+ body = {
291
+ "model": spec["model_id"],
292
+ "input": json.dumps(state, ensure_ascii=False),
293
+ "questions": _openai_questions(),
294
+ }
295
+ request = urllib.request.Request(
296
+ "https://api.openai.com/v1/decisions",
297
+ data=json.dumps(body).encode(),
298
+ headers={
299
+ "Authorization": f"Bearer {auth}",
300
+ "Content-Type": "application/json",
301
+ },
302
+ method="POST",
303
+ )
304
+ with urllib.request.urlopen(request, timeout=90) as response:
305
+ payload = json.load(response)
306
+ return _openai_answers(payload)
307
+
308
+ raise RuntimeError(f"Unsupported decision provider: {provider}")
309
+
310
+
311
+ def _retry(auth, state: dict, limiter: RateLimiter, spec: dict[str, str]) -> dict:
312
+ last_error: Exception | None = None
313
+ for attempt in range(6):
314
+ limiter.wait()
315
+ try:
316
+ return _post(auth, state, spec)
317
+ except urllib.error.HTTPError as error:
318
+ detail = error.read().decode(errors="replace")[:500]
319
+ if error.code == 429 or error.code >= 500:
320
+ retry_after = error.headers.get("Retry-After")
321
+ delay = float(retry_after) if retry_after else 2**attempt
322
+ time.sleep(min(delay, 60))
323
+ last_error = RuntimeError(f"HTTP {error.code}: {detail}")
324
+ continue
325
+ raise RuntimeError(f"HTTP {error.code}: {detail}") from error
326
+ except (urllib.error.URLError, TimeoutError) as error:
327
+ last_error = error
328
+ time.sleep(2**attempt)
329
+ raise RuntimeError(
330
+ f"{spec['name']} failed after retries: {last_error}"
331
+ )
332
+
333
+
334
+ def _pending(
335
+ con: duckdb.DuckDBPyConnection, limit: int | None
336
+ ) -> list[tuple[str, str, str, str]]:
337
+ query = """
338
+ SELECT e.tool_name, e.action_type, e.target, min(e.raw_target) AS raw_example
339
+ FROM tool_events e
340
+ LEFT JOIN target_labels l
341
+ ON e.tool_name = l.tool_name
342
+ AND e.action_type = l.action_type
343
+ AND e.target = l.target
344
+ WHERE l.tool_name IS NULL
345
+ GROUP BY 1, 2, 3
346
+ ORDER BY count(*) DESC
347
+ """
348
+ if limit is not None:
349
+ query += f" LIMIT {int(limit)}"
350
+ return con.execute(query).fetchall()
351
+
352
+
353
+ def _state(row: tuple[str, str, str, str]) -> dict:
354
+ tool_name, action_type, target, raw_example = row
355
+ return {
356
+ "tool": tool_name,
357
+ "action_type": action_type,
358
+ "normalized_target": target[:500],
359
+ "raw_example": (raw_example or "").replace("\x00", " ")[:500],
360
+ }
361
+
362
+
363
+ def label_targets(
364
+ db_path: str | Path = DB_PATH,
365
+ limit: int | None = None,
366
+ rpm: int = DEFAULT_RPM,
367
+ workers: int = DEFAULT_WORKERS,
368
+ model: str | None = None,
369
+ ) -> tuple[int, int]:
370
+ con = connect(db_path)
371
+ rows = _pending(con, limit)
372
+ if not rows:
373
+ con.close()
374
+ return 0, 0
375
+
376
+ spec = _resolve_model(model)
377
+ auth = _auth_for(spec)
378
+ limiter = RateLimiter(rpm)
379
+ done = 0
380
+ failed = 0
381
+ batch: list[tuple] = []
382
+ started = time.monotonic()
383
+
384
+ def work(row: tuple[str, str, str, str]) -> tuple:
385
+ result = _retry(auth, _state(row), limiter, spec)
386
+ intent = result["intent"]
387
+ object_type = result["object_type"]
388
+ return (
389
+ row[0],
390
+ row[1],
391
+ row[2],
392
+ intent["choice"],
393
+ float(intent.get("confidence") or 0.0),
394
+ object_type["choice"],
395
+ float(object_type.get("confidence") or 0.0),
396
+ spec["name"],
397
+ )
398
+
399
+ try:
400
+ with ThreadPoolExecutor(max_workers=workers) as pool:
401
+ futures = {pool.submit(work, row): row for row in rows}
402
+ for future in as_completed(futures):
403
+ row = futures[future]
404
+ try:
405
+ batch.append(future.result())
406
+ done += 1
407
+ except Exception as error: # noqa: BLE001 - report per-row failures
408
+ failed += 1
409
+ print(f"FAILED {row[0]} {row[2][:80]}: {error}")
410
+ if len(batch) >= 100:
411
+ con.executemany(
412
+ "INSERT OR REPLACE INTO target_labels VALUES (?, ?, ?, ?, ?, ?, ?, ?, current_timestamp)",
413
+ batch,
414
+ )
415
+ batch.clear()
416
+ if (done + failed) % 100 == 0:
417
+ elapsed = time.monotonic() - started
418
+ rate = (done + failed) / elapsed * 60 if elapsed else 0
419
+ print(
420
+ f"{done + failed}/{len(rows)} ok={done} failed={failed} "
421
+ f"{rate:.0f}/min",
422
+ flush=True,
423
+ )
424
+ if batch:
425
+ con.executemany(
426
+ "INSERT OR REPLACE INTO target_labels VALUES (?, ?, ?, ?, ?, ?, ?, ?, current_timestamp)",
427
+ batch,
428
+ )
429
+ finally:
430
+ con.close()
431
+ return done, failed
trace2rules/models.py ADDED
@@ -0,0 +1,33 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass, field
4
+
5
+
6
+ @dataclass(slots=True)
7
+ class ToolCall:
8
+ """A normalized tool call before it is classified for DuckDB."""
9
+
10
+ tool_name: str
11
+ raw_target: str
12
+ observation: str = ""
13
+ action_type: str | None = None
14
+ target: str | None = None
15
+ verb: str | None = None
16
+ object_id: str | None = None
17
+ exit_code: int | None = None
18
+ is_error: bool | None = None
19
+ segment_idx: int = 0
20
+
21
+
22
+ @dataclass(slots=True)
23
+ class Session:
24
+ """A normalized session accepted by the shared DuckDB writer."""
25
+
26
+ session_id: str
27
+ repo: str
28
+ instance_id: str = ""
29
+ resolved: bool = False
30
+ calls: list[ToolCall] = field(default_factory=list)
31
+ source: str = ""
32
+ agent: str = ""
33
+ success_score: float | None = None
@@ -0,0 +1,159 @@
1
+ from __future__ import annotations
2
+
3
+ from collections import defaultdict
4
+ from dataclasses import dataclass
5
+
6
+ import duckdb
7
+
8
+
9
+ @dataclass(frozen=True)
10
+ class SequencePattern:
11
+ sequence: str
12
+ sessions: int
13
+ occurrences: int
14
+ tokens: int
15
+ repos: tuple[str, ...]
16
+ example: str
17
+
18
+
19
+ def _token(
20
+ action: str,
21
+ object_id: str,
22
+ outcome: str,
23
+ intent: str,
24
+ object_type: str,
25
+ intent_confidence: float,
26
+ object_confidence: float,
27
+ min_intent: float,
28
+ min_object: float,
29
+ ) -> str:
30
+ obj = object_id or "?"
31
+ obj = " ".join(obj.split())
32
+ if len(obj) > 140:
33
+ obj = obj[:137] + "..."
34
+ if (
35
+ intent
36
+ and intent != "other"
37
+ and object_type
38
+ and object_type != "unknown"
39
+ and intent_confidence >= min_intent
40
+ and object_confidence >= min_object
41
+ ):
42
+ return f"{intent}/{object_type}:{obj}:{outcome}"
43
+ return f"{action}:{obj}:{outcome}"
44
+
45
+
46
+ def mine_sequences(
47
+ con: duckdb.DuckDBPyConnection,
48
+ min_sessions: int = 3,
49
+ max_len: int = 4,
50
+ top: int = 25,
51
+ min_confidence: float = 0.4,
52
+ ) -> list[SequencePattern]:
53
+ rows = con.execute(
54
+ """
55
+ SELECT e.session_id, e.step_idx, e.segment_idx, e.repo,
56
+ e.action_type, e.object_id, e.outcome, e.approx_out_tokens,
57
+ e.raw_target, l.intent, l.object_type,
58
+ l.intent_confidence, l.object_confidence
59
+ FROM tool_events e
60
+ LEFT JOIN target_labels l
61
+ ON e.tool_name = l.tool_name
62
+ AND e.action_type = l.action_type
63
+ AND e.target = l.target
64
+ WHERE e.tool_name NOT IN ('think', 'finish', 'task_tracker')
65
+ AND e.action_type <> 'other'
66
+ ORDER BY e.session_id, e.step_idx
67
+ """
68
+ ).fetchall()
69
+
70
+ sessions: dict[str, list[tuple]] = defaultdict(list)
71
+ for row in rows:
72
+ sessions[row[0]].append(row)
73
+
74
+ aggregates: dict[tuple[str, ...], dict] = {}
75
+ object_floor = max(min_confidence, 0.6)
76
+ for session_id, events in sessions.items():
77
+ tokens = [
78
+ _token(
79
+ event[4],
80
+ event[5]
81
+ if event[5] not in {"", "<path>", "unknown", "?"}
82
+ else (event[8] or "?"),
83
+ event[6],
84
+ event[9] or "",
85
+ event[10] or "",
86
+ float(event[11] or 0.0),
87
+ float(event[12] or 0.0),
88
+ min_confidence,
89
+ object_floor,
90
+ )
91
+ for event in events
92
+ ]
93
+ for length in range(2, max_len + 1):
94
+ for start in range(len(events) - length + 1):
95
+ window = events[start : start + length]
96
+ if any(
97
+ window[index][1] + 1 != window[index + 1][1]
98
+ or window[index][2] != window[index + 1][2]
99
+ for index in range(len(window) - 1)
100
+ ):
101
+ continue
102
+ window_tokens = tuple(tokens[start : start + length])
103
+ if not any(event[6] == "err" for event in window):
104
+ continue
105
+ agg = aggregates.setdefault(
106
+ window_tokens,
107
+ {
108
+ "sessions": set(),
109
+ "occurrences": 0,
110
+ "tokens": 0,
111
+ "repos": set(),
112
+ "example": "",
113
+ },
114
+ )
115
+ agg["sessions"].add(session_id)
116
+ agg["repos"].update(event[3] for event in window)
117
+ agg["occurrences"] += 1
118
+ agg["tokens"] += sum(
119
+ event[7] for event in window if event[6] == "err"
120
+ )
121
+ if not agg["example"]:
122
+ agg["example"] = " -> ".join(
123
+ (event[8] or event[5] or "").replace("\x00", " ")
124
+ for event in window
125
+ )
126
+
127
+ candidates = [
128
+ SequencePattern(
129
+ sequence=" -> ".join(tokens),
130
+ sessions=len(value["sessions"]),
131
+ occurrences=value["occurrences"],
132
+ tokens=value["tokens"],
133
+ repos=tuple(sorted(value["repos"])),
134
+ example=value["example"],
135
+ )
136
+ for tokens, value in aggregates.items()
137
+ if len(value["sessions"]) >= min_sessions
138
+ ]
139
+
140
+ # Prefer maximal windows; a shorter prefix with the same support is redundant.
141
+ candidates.sort(key=lambda pattern: len(pattern.sequence.split(" -> ")), reverse=True)
142
+ kept: list[SequencePattern] = []
143
+ for pattern in candidates:
144
+ if any(
145
+ pattern.sequence in longer.sequence
146
+ and pattern.sessions <= longer.sessions
147
+ for longer in kept
148
+ ):
149
+ continue
150
+ object_ids = {
151
+ token.rsplit(":", 2)[1]
152
+ for token in pattern.sequence.split(" -> ")
153
+ }
154
+ if len(object_ids) == 1 and pattern.sessions < 5:
155
+ continue
156
+ kept.append(pattern)
157
+
158
+ kept.sort(key=lambda pattern: (pattern.sessions, pattern.tokens), reverse=True)
159
+ return kept[:top]
@@ -0,0 +1 @@
1
+ """Session sources that produce canonical trace2rules records."""