trace2rules 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trace2rules/__init__.py +1 -0
- trace2rules/cli.py +543 -0
- trace2rules/cluster.py +348 -0
- trace2rules/db.py +84 -0
- trace2rules/detectors.py +183 -0
- trace2rules/ingest.py +717 -0
- trace2rules/labels.py +431 -0
- trace2rules/models.py +33 -0
- trace2rules/patterns.py +159 -0
- trace2rules/sources/__init__.py +1 -0
- trace2rules/sources/agent_trajectory.py +456 -0
- trace2rules/sources/common.py +99 -0
- trace2rules/sources/opencode.py +154 -0
- trace2rules/sources/swe_chat.py +117 -0
- trace2rules/synthesize.py +538 -0
- trace2rules-0.1.0.dist-info/METADATA +323 -0
- trace2rules-0.1.0.dist-info/RECORD +20 -0
- trace2rules-0.1.0.dist-info/WHEEL +4 -0
- trace2rules-0.1.0.dist-info/entry_points.txt +3 -0
- trace2rules-0.1.0.dist-info/licenses/LICENSE +21 -0
trace2rules/ingest.py
ADDED
|
@@ -0,0 +1,717 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
import shlex
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import duckdb
|
|
12
|
+
|
|
13
|
+
from .db import DB_PATH, connect
|
|
14
|
+
from .models import Session, ToolCall
|
|
15
|
+
|
|
16
|
+
SOURCE = "hf://datasets/nebius/SWE-rebench-openhands-trajectories/trajectories.parquet"
|
|
17
|
+
DEFAULT_REPOS = ["django__django", "pytest-dev__pytest"]
|
|
18
|
+
|
|
19
|
+
EXIT_RE = re.compile(
|
|
20
|
+
r"\[(?:the command completed|command finished) with exit code (-?\d+)\]",
|
|
21
|
+
re.IGNORECASE,
|
|
22
|
+
)
|
|
23
|
+
EXIT_FALLBACK_RE = re.compile(r"(?:Exit code|exit code):\s*(-?\d+)")
|
|
24
|
+
TEXT_ERROR_RE = re.compile(r"Traceback|Error:|ModuleNotFoundError|No such file or directory")
|
|
25
|
+
TOOL_ERROR_RE = re.compile(
|
|
26
|
+
r"Invalid\s+`?path`?|File not found|No such file or directory|does not exist|Permission denied",
|
|
27
|
+
re.IGNORECASE,
|
|
28
|
+
)
|
|
29
|
+
TIMEOUT_RE = re.compile(r"timed out|Timeout", re.IGNORECASE)
|
|
30
|
+
|
|
31
|
+
ENV_PREFIX_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=\S*\s+")
|
|
32
|
+
CD_PREFIX_RE = re.compile(r"^cd\s+[^&;]+\s*(?:&&|;)\s*")
|
|
33
|
+
EXPORT_PREFIX_RE = re.compile(r"^export\s+[^&;]+\s*(?:&&|;)\s*")
|
|
34
|
+
EDIT_VERBS = {"apply_patch", "patch"}
|
|
35
|
+
READ_VERBS = {"cat", "head", "tail", "less", "more", "wc", "nl", "ls", "tree", "stat", "file", "du", "sed", "awk"}
|
|
36
|
+
SEARCH_VERBS = {"grep", "rg", "ag", "ack", "fd", "find"}
|
|
37
|
+
EDITOR_READ_COMMANDS = {"view", "show"}
|
|
38
|
+
FLAG_WITH_VALUE = {"-e", "-A", "-B", "-C", "-m", "-g", "-t", "--include", "--exclude", "--glob", "--type", "-name", "-path", "-type"}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class IngestStats:
|
|
43
|
+
prefix: str
|
|
44
|
+
matched: int
|
|
45
|
+
sessions: int
|
|
46
|
+
events: int
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _is_path_token(token: str) -> bool:
|
|
50
|
+
if token.startswith("-") or token in {"=", "&&", ";", "|"}:
|
|
51
|
+
return False
|
|
52
|
+
if token.startswith(("2>", "1>", "&>")) or any(char in token for char in "<>|&"):
|
|
53
|
+
return False
|
|
54
|
+
if token.startswith(("skill:", "xd:")):
|
|
55
|
+
return False
|
|
56
|
+
if "/" in token:
|
|
57
|
+
return True
|
|
58
|
+
return bool(re.search(r"\.(py|pyi|txt|md|rst|toml|cfg|ini|yaml|yml|json)$", token))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def normalize_path(path: str, instance_id: str = "") -> str:
|
|
62
|
+
path = (path or "").strip().strip("'\"").rstrip(";,)")
|
|
63
|
+
if not path:
|
|
64
|
+
return ""
|
|
65
|
+
if path.startswith(("skill:", "xd:")):
|
|
66
|
+
return ""
|
|
67
|
+
path = path.replace("\\", "/")
|
|
68
|
+
if path.rstrip("/") in ("/workspace", "/testbed", "/repo", "/app"):
|
|
69
|
+
return "."
|
|
70
|
+
for root in ("/workspace/", "/testbed/", "/repo/", "/app/"):
|
|
71
|
+
if path.startswith(root):
|
|
72
|
+
path = path[len(root) :]
|
|
73
|
+
workspace_root = root == "/workspace/"
|
|
74
|
+
break
|
|
75
|
+
else:
|
|
76
|
+
workspace_root = False
|
|
77
|
+
parts = [p for p in path.split("/") if p not in ("", ".")]
|
|
78
|
+
if workspace_root and parts and (len(parts) > 1 or "__" in parts[0]):
|
|
79
|
+
parts = parts[1:]
|
|
80
|
+
elif parts and ("__" in parts[0] or parts[0] == instance_id):
|
|
81
|
+
parts = parts[1:]
|
|
82
|
+
path = "/".join(parts)
|
|
83
|
+
return path or "."
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _strip_shell_prefix(command: str) -> str:
|
|
87
|
+
command = command.strip()
|
|
88
|
+
while True:
|
|
89
|
+
before = command
|
|
90
|
+
command = ENV_PREFIX_RE.sub("", command)
|
|
91
|
+
command = CD_PREFIX_RE.sub("", command)
|
|
92
|
+
command = EXPORT_PREFIX_RE.sub("", command)
|
|
93
|
+
if command == before:
|
|
94
|
+
return command
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _first_verb(command: str) -> tuple[str, list[str]]:
|
|
98
|
+
command = _strip_shell_prefix(command)
|
|
99
|
+
try:
|
|
100
|
+
tokens = shlex.split(command, posix=True)
|
|
101
|
+
except ValueError:
|
|
102
|
+
tokens = command.split()
|
|
103
|
+
if not tokens:
|
|
104
|
+
return "", []
|
|
105
|
+
if tokens[0] == "git" and len(tokens) > 1:
|
|
106
|
+
return f"git {tokens[1]}", tokens[2:]
|
|
107
|
+
return tokens[0], tokens[1:]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _basename(path: str) -> str:
|
|
111
|
+
path = (path or "").rstrip("/")
|
|
112
|
+
return path.rsplit("/", 1)[-1] if path else ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _shell_verb_tokens(command: str) -> tuple[str, list[str]]:
|
|
116
|
+
verb, tokens = _first_verb(command)
|
|
117
|
+
if verb in ("python", "python3"):
|
|
118
|
+
if "pytest" in tokens:
|
|
119
|
+
return "pytest", tokens
|
|
120
|
+
if "-m" in tokens:
|
|
121
|
+
index = tokens.index("-m")
|
|
122
|
+
module = tokens[index + 1] if index + 1 < len(tokens) else ""
|
|
123
|
+
if module in ("pytest", "pip"):
|
|
124
|
+
return module, tokens
|
|
125
|
+
return module or verb, tokens
|
|
126
|
+
if verb.startswith("git "):
|
|
127
|
+
return "git", [verb.split(" ", 1)[1], *tokens]
|
|
128
|
+
if verb.startswith("pip"):
|
|
129
|
+
return "pip", tokens
|
|
130
|
+
return verb, tokens
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _find_object(tokens: list[str]) -> str:
|
|
134
|
+
path = ""
|
|
135
|
+
name = ""
|
|
136
|
+
index = 0
|
|
137
|
+
while index < len(tokens):
|
|
138
|
+
token = tokens[index]
|
|
139
|
+
if token in ("-name", "-iname") and index + 1 < len(tokens):
|
|
140
|
+
name = tokens[index + 1]
|
|
141
|
+
index += 2
|
|
142
|
+
continue
|
|
143
|
+
if not path and not token.startswith("-"):
|
|
144
|
+
path = token
|
|
145
|
+
index += 1
|
|
146
|
+
if name:
|
|
147
|
+
if path:
|
|
148
|
+
path = normalize_path(path).rstrip("/")
|
|
149
|
+
return f"{path}/{name}" if path not in ("", ".") else name
|
|
150
|
+
return normalize_path(path) if path else "."
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _object_id(
|
|
154
|
+
action: str, verb: str, tokens: list[str], raw_command: str, target: str
|
|
155
|
+
) -> str:
|
|
156
|
+
if action == "search" and verb == "find":
|
|
157
|
+
return _find_object(tokens)
|
|
158
|
+
if action in ("read", "edit", "search") and " " not in target:
|
|
159
|
+
if not target or target == ".":
|
|
160
|
+
return "" if action == "search" else "."
|
|
161
|
+
return _basename(target) if action in ("read", "edit") else target
|
|
162
|
+
|
|
163
|
+
if verb == "python" and "-c" in tokens:
|
|
164
|
+
payload = raw_command.split("-c", 1)[1].strip() if "-c" in raw_command else ""
|
|
165
|
+
match = re.search(r"\b(?:import|from)\s+([A-Za-z_][\w.]*)", payload)
|
|
166
|
+
if match:
|
|
167
|
+
return match.group(1)
|
|
168
|
+
return "inline:" + hashlib.sha1(payload.encode()).hexdigest()[:8]
|
|
169
|
+
|
|
170
|
+
if verb == "pytest":
|
|
171
|
+
paths = [token for token in tokens if _is_path_token(token)]
|
|
172
|
+
return _basename(paths[-1]) if paths else "pytest"
|
|
173
|
+
|
|
174
|
+
if verb == "python":
|
|
175
|
+
if "-m" in tokens:
|
|
176
|
+
index = tokens.index("-m")
|
|
177
|
+
return tokens[index + 1] if index + 1 < len(tokens) else "python"
|
|
178
|
+
for token in tokens:
|
|
179
|
+
if _is_path_token(token):
|
|
180
|
+
return _basename(token)
|
|
181
|
+
return ""
|
|
182
|
+
|
|
183
|
+
if verb == "pip" and "install" in tokens:
|
|
184
|
+
args = tokens[tokens.index("install") + 1 :]
|
|
185
|
+
if "-e" in args:
|
|
186
|
+
index = args.index("-e")
|
|
187
|
+
destination = args[index + 1] if index + 1 < len(args) else ""
|
|
188
|
+
return "editable" if destination in (".", "./") else _basename(destination)
|
|
189
|
+
packages = [arg for arg in args if not arg.startswith("-")]
|
|
190
|
+
return ",".join(packages[:2]) or "install"
|
|
191
|
+
if verb == "pip":
|
|
192
|
+
return ""
|
|
193
|
+
|
|
194
|
+
if verb == "git":
|
|
195
|
+
return tokens[0] if tokens else ""
|
|
196
|
+
if verb == "export":
|
|
197
|
+
match = re.search(r"\bexport\s+([A-Za-z_][A-Za-z0-9_]*)", raw_command)
|
|
198
|
+
return match.group(1) if match else ""
|
|
199
|
+
if action == "exec" and target and " " not in target:
|
|
200
|
+
return _basename(target)
|
|
201
|
+
return ""
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _extract_exit_code(observation: str) -> int | None:
|
|
205
|
+
match = EXIT_RE.search(observation) or EXIT_FALLBACK_RE.search(observation)
|
|
206
|
+
return int(match.group(1)) if match else None
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _error_signature(
|
|
210
|
+
observation: str,
|
|
211
|
+
exit_code: int | None,
|
|
212
|
+
timeout: bool,
|
|
213
|
+
text_error: bool,
|
|
214
|
+
tool_error: bool,
|
|
215
|
+
) -> str:
|
|
216
|
+
if timeout:
|
|
217
|
+
return "timeout"
|
|
218
|
+
lowered = observation.lower()
|
|
219
|
+
if "did not appear verbatim" in lowered:
|
|
220
|
+
return "editor:verbatim"
|
|
221
|
+
if "invalid `path`" in lowered or "invalid path" in lowered:
|
|
222
|
+
return "editor:invalid_path"
|
|
223
|
+
matches = re.findall(
|
|
224
|
+
r"(?:^|\n)\s*(?:E\s+)?([A-Za-z_][A-Za-z0-9_.]*(?:Error|Exception))(?::|\s)",
|
|
225
|
+
observation,
|
|
226
|
+
)
|
|
227
|
+
if matches:
|
|
228
|
+
return f"exception:{matches[-1]}"
|
|
229
|
+
if exit_code not in (None, 0):
|
|
230
|
+
return f"exit:{exit_code}"
|
|
231
|
+
if text_error:
|
|
232
|
+
return "text_error"
|
|
233
|
+
if tool_error:
|
|
234
|
+
return "tool_error"
|
|
235
|
+
return ""
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _event_outcome(
|
|
239
|
+
action: str,
|
|
240
|
+
observation: str,
|
|
241
|
+
exit_code: int | None,
|
|
242
|
+
text_error: bool,
|
|
243
|
+
timeout: bool,
|
|
244
|
+
tool_error: bool,
|
|
245
|
+
) -> str:
|
|
246
|
+
if exit_code is not None:
|
|
247
|
+
return "ok" if exit_code == 0 else "err"
|
|
248
|
+
if timeout:
|
|
249
|
+
return "timeout"
|
|
250
|
+
if action == "edit":
|
|
251
|
+
if tool_error or text_error:
|
|
252
|
+
return "err"
|
|
253
|
+
return "ok" if observation else "unknown"
|
|
254
|
+
if action == "exec" and text_error:
|
|
255
|
+
return "err"
|
|
256
|
+
if action in ("read", "search"):
|
|
257
|
+
if tool_error:
|
|
258
|
+
return "err"
|
|
259
|
+
return "ok" if observation else "unknown"
|
|
260
|
+
return "unknown"
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _path_args(verb: str, tokens: list[str]) -> list[str]:
|
|
264
|
+
if verb == "find":
|
|
265
|
+
for token in tokens:
|
|
266
|
+
if not token.startswith("-"):
|
|
267
|
+
return [token]
|
|
268
|
+
return []
|
|
269
|
+
out: list[str] = []
|
|
270
|
+
skip_next = False
|
|
271
|
+
pattern_token: str | None = None
|
|
272
|
+
for token in tokens:
|
|
273
|
+
if skip_next:
|
|
274
|
+
skip_next = False
|
|
275
|
+
continue
|
|
276
|
+
if token in FLAG_WITH_VALUE:
|
|
277
|
+
skip_next = True
|
|
278
|
+
continue
|
|
279
|
+
if token.startswith("-"):
|
|
280
|
+
continue
|
|
281
|
+
if verb in {"grep", "rg", "ag", "ack", "git grep"} and pattern_token is None:
|
|
282
|
+
pattern_token = token
|
|
283
|
+
continue
|
|
284
|
+
if _is_path_token(token):
|
|
285
|
+
out.append(token)
|
|
286
|
+
return out
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _classify_shell(command: str, instance_id: str) -> tuple[str, str]:
|
|
290
|
+
verb, tokens = _first_verb(command)
|
|
291
|
+
lower = command.lower()
|
|
292
|
+
if verb in EDIT_VERBS or lower.startswith("git apply") or re.search(r"\bsed\s+-i|\bperl\s+-[a-z]*i", lower):
|
|
293
|
+
paths = _path_args(verb, tokens)
|
|
294
|
+
return "edit", normalize_path(paths[-1], instance_id) if paths else _parameterize(command, instance_id)
|
|
295
|
+
if verb in SEARCH_VERBS:
|
|
296
|
+
paths = _path_args(verb, tokens)
|
|
297
|
+
return "search", normalize_path(paths[0], instance_id) if paths else ""
|
|
298
|
+
if verb in READ_VERBS:
|
|
299
|
+
paths = _path_args(verb, tokens)
|
|
300
|
+
return "read", normalize_path(paths[0], instance_id) if paths else "."
|
|
301
|
+
if verb in {"git diff", "git log", "git status", "git show", "git blame"}:
|
|
302
|
+
return "read", verb
|
|
303
|
+
return "exec", _parameterize(command, instance_id)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _parameterize(command: str, instance_id: str) -> str:
|
|
307
|
+
command = _strip_shell_prefix(" ".join((command or "").split()))
|
|
308
|
+
if not command:
|
|
309
|
+
return ""
|
|
310
|
+
try:
|
|
311
|
+
tokens = shlex.split(command, posix=True)
|
|
312
|
+
except ValueError:
|
|
313
|
+
tokens = command.split()
|
|
314
|
+
|
|
315
|
+
out: list[str] = []
|
|
316
|
+
i = 0
|
|
317
|
+
while i < len(tokens):
|
|
318
|
+
token = tokens[i]
|
|
319
|
+
if token == "-k" and i + 1 < len(tokens):
|
|
320
|
+
out.extend([token, "<arg>"])
|
|
321
|
+
i += 2
|
|
322
|
+
continue
|
|
323
|
+
if _is_path_token(token):
|
|
324
|
+
out.append("<path>")
|
|
325
|
+
else:
|
|
326
|
+
out.append(token)
|
|
327
|
+
i += 1
|
|
328
|
+
result = " ".join(out)
|
|
329
|
+
return result[:240]
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _classify(tool_name: str, raw_target: str, instance_id: str) -> tuple[str, str, str, str]:
|
|
333
|
+
if tool_name == "str_replace_editor":
|
|
334
|
+
command = raw_target.split("\x00", 1)[0]
|
|
335
|
+
path = raw_target.split("\x00", 1)[1] if "\x00" in raw_target else ""
|
|
336
|
+
action = "read" if command in EDITOR_READ_COMMANDS else "edit"
|
|
337
|
+
target = normalize_path(path, instance_id)
|
|
338
|
+
return action, target, action, _basename(target) if target != "." else "."
|
|
339
|
+
action, target = _classify_shell(raw_target, instance_id)
|
|
340
|
+
verb, tokens = _shell_verb_tokens(raw_target)
|
|
341
|
+
object_id = _object_id(action, verb, tokens, raw_target, target)
|
|
342
|
+
return action, target, verb, object_id
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _tool_call_parts(tool_name: str, arguments: str) -> tuple[str, str]:
|
|
346
|
+
try:
|
|
347
|
+
args = json.loads(arguments or "{}")
|
|
348
|
+
except json.JSONDecodeError:
|
|
349
|
+
return tool_name, ""
|
|
350
|
+
if tool_name == "str_replace_editor":
|
|
351
|
+
command = str(args.get("command") or args.get("action") or "")
|
|
352
|
+
path = str(args.get("path") or args.get("file") or "")
|
|
353
|
+
return f"{command}\x00{path}", path or command
|
|
354
|
+
command = str(args.get("command") or args.get("cmd") or "")
|
|
355
|
+
return command, command
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _observation_for_call(call_id: str, obs_by_id: dict[str, str], rows: list[dict], index: int) -> str:
|
|
359
|
+
if call_id and call_id in obs_by_id:
|
|
360
|
+
return obs_by_id[call_id]
|
|
361
|
+
for msg in rows[index + 1 :]:
|
|
362
|
+
if msg["role"] == "tool":
|
|
363
|
+
return msg["content"] or ""
|
|
364
|
+
if msg["role"] == "assistant":
|
|
365
|
+
break
|
|
366
|
+
return ""
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _parse_row(row: tuple) -> tuple[dict, list[dict]]:
|
|
370
|
+
trajectory_id, instance_id, repo, resolved, trajectory = row
|
|
371
|
+
messages = list(trajectory or [])
|
|
372
|
+
obs_by_id = {
|
|
373
|
+
m["tool_call_id"]: (m["content"] or "") for m in messages if m["role"] == "tool"
|
|
374
|
+
}
|
|
375
|
+
calls: list[dict] = []
|
|
376
|
+
step = 0
|
|
377
|
+
for index, msg in enumerate(messages):
|
|
378
|
+
for tool_call in msg.get("tool_calls") or []:
|
|
379
|
+
fn = tool_call["function"]
|
|
380
|
+
raw_target, _ = _tool_call_parts(fn["name"], fn.get("arguments", ""))
|
|
381
|
+
observation = _observation_for_call(tool_call.get("id", ""), obs_by_id, messages, index)
|
|
382
|
+
exit_code = _extract_exit_code(observation)
|
|
383
|
+
text_error = bool(TEXT_ERROR_RE.search(observation))
|
|
384
|
+
timeout = bool(TIMEOUT_RE.search(observation))
|
|
385
|
+
tool_error = len(observation) < 500 and bool(TOOL_ERROR_RE.search(observation))
|
|
386
|
+
calls.append(
|
|
387
|
+
{
|
|
388
|
+
"event_id": f"{trajectory_id}:{step}",
|
|
389
|
+
"session_id": trajectory_id,
|
|
390
|
+
"repo": repo,
|
|
391
|
+
"step_idx": step,
|
|
392
|
+
"tool_name": fn["name"],
|
|
393
|
+
"raw_target": raw_target,
|
|
394
|
+
"obs": observation,
|
|
395
|
+
"exit_code": exit_code,
|
|
396
|
+
"text_error": text_error,
|
|
397
|
+
"timeout": timeout,
|
|
398
|
+
"tool_error": tool_error,
|
|
399
|
+
"error_signature": _error_signature(
|
|
400
|
+
observation, exit_code, timeout, text_error, tool_error
|
|
401
|
+
),
|
|
402
|
+
}
|
|
403
|
+
)
|
|
404
|
+
step += 1
|
|
405
|
+
session = {
|
|
406
|
+
"session_id": trajectory_id,
|
|
407
|
+
"repo": repo,
|
|
408
|
+
"instance_id": instance_id,
|
|
409
|
+
"resolved": bool(resolved),
|
|
410
|
+
"total_turns": step,
|
|
411
|
+
}
|
|
412
|
+
return session, calls
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _stream_rows(source: str, prefix: str, limit: int):
|
|
416
|
+
reader = duckdb.connect(
|
|
417
|
+
config={
|
|
418
|
+
"memory_limit": "4500MB",
|
|
419
|
+
"threads": "1",
|
|
420
|
+
"preserve_insertion_order": "false",
|
|
421
|
+
}
|
|
422
|
+
)
|
|
423
|
+
try:
|
|
424
|
+
if source.startswith(("hf://", "http://", "https://")):
|
|
425
|
+
try:
|
|
426
|
+
reader.execute("LOAD httpfs")
|
|
427
|
+
except duckdb.Error:
|
|
428
|
+
reader.execute("INSTALL httpfs")
|
|
429
|
+
reader.execute("LOAD httpfs")
|
|
430
|
+
limit_sql = f"LIMIT {int(limit)}" if limit else ""
|
|
431
|
+
query = f"""
|
|
432
|
+
SELECT trajectory_id, instance_id, repo, resolved, trajectory
|
|
433
|
+
FROM read_parquet(?)
|
|
434
|
+
WHERE starts_with(instance_id, ?)
|
|
435
|
+
{limit_sql}
|
|
436
|
+
"""
|
|
437
|
+
cursor = reader.execute(query, [source, prefix])
|
|
438
|
+
while True:
|
|
439
|
+
batch = cursor.fetchmany(16)
|
|
440
|
+
if not batch:
|
|
441
|
+
return
|
|
442
|
+
yield from batch
|
|
443
|
+
finally:
|
|
444
|
+
reader.close()
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def ingest(
|
|
448
|
+
repos: list[str] | None = None,
|
|
449
|
+
limit: int = 200,
|
|
450
|
+
db_path: str | Path = DB_PATH,
|
|
451
|
+
source: str = SOURCE,
|
|
452
|
+
) -> list[IngestStats]:
|
|
453
|
+
repos = repos or DEFAULT_REPOS
|
|
454
|
+
con = connect(db_path)
|
|
455
|
+
con.execute("CREATE OR REPLACE TEMP TABLE new_sessions (session_id VARCHAR, repo VARCHAR, instance_id VARCHAR, resolved BOOLEAN, total_turns INTEGER)")
|
|
456
|
+
con.execute(
|
|
457
|
+
"CREATE OR REPLACE TEMP TABLE raw_calls (event_id VARCHAR, session_id VARCHAR, repo VARCHAR, step_idx INTEGER, tool_name VARCHAR, raw_target VARCHAR, obs_content VARCHAR, exit_code INTEGER, text_error BOOLEAN, timeout BOOLEAN, tool_error BOOLEAN, error_signature VARCHAR)"
|
|
458
|
+
)
|
|
459
|
+
stats: list[IngestStats] = []
|
|
460
|
+
try:
|
|
461
|
+
for prefix in repos:
|
|
462
|
+
matched = 0
|
|
463
|
+
rows_for_prefix: list[dict] = []
|
|
464
|
+
new_sessions_batch: list[dict] = []
|
|
465
|
+
for row in _stream_rows(source, prefix, limit):
|
|
466
|
+
matched += 1
|
|
467
|
+
session, calls = _parse_row(row)
|
|
468
|
+
new_sessions_batch.append(session)
|
|
469
|
+
rows_for_prefix.extend(calls)
|
|
470
|
+
if len(rows_for_prefix) >= 2_000:
|
|
471
|
+
_flush_calls(con, rows_for_prefix)
|
|
472
|
+
rows_for_prefix.clear()
|
|
473
|
+
if len(new_sessions_batch) >= 64:
|
|
474
|
+
_flush_sessions(con, new_sessions_batch)
|
|
475
|
+
new_sessions_batch.clear()
|
|
476
|
+
_flush_calls(con, rows_for_prefix)
|
|
477
|
+
_flush_sessions(con, new_sessions_batch)
|
|
478
|
+
con.execute("DELETE FROM tool_events WHERE session_id IN (SELECT session_id FROM new_sessions)")
|
|
479
|
+
con.execute("DELETE FROM sessions WHERE session_id IN (SELECT session_id FROM new_sessions)")
|
|
480
|
+
con.execute(
|
|
481
|
+
"INSERT INTO sessions (session_id, repo, instance_id, resolved, total_turns) "
|
|
482
|
+
"SELECT session_id, repo, instance_id, resolved, total_turns FROM new_sessions"
|
|
483
|
+
)
|
|
484
|
+
_build_target_map(con)
|
|
485
|
+
rows = con.execute(
|
|
486
|
+
"""
|
|
487
|
+
SELECT r.event_id, r.session_id, r.repo, r.step_idx, r.tool_name,
|
|
488
|
+
m.action_type, m.verb, m.object_id, m.target, r.raw_target,
|
|
489
|
+
r.exit_code, r.text_error, r.timeout, r.tool_error,
|
|
490
|
+
r.error_signature, r.obs_content
|
|
491
|
+
FROM raw_calls r JOIN target_map m USING (tool_name, raw_target)
|
|
492
|
+
"""
|
|
493
|
+
).fetchall()
|
|
494
|
+
events = []
|
|
495
|
+
for row in rows:
|
|
496
|
+
(
|
|
497
|
+
event_id,
|
|
498
|
+
session_id,
|
|
499
|
+
event_repo,
|
|
500
|
+
step_idx,
|
|
501
|
+
tool_name,
|
|
502
|
+
action_type,
|
|
503
|
+
verb,
|
|
504
|
+
object_id,
|
|
505
|
+
target,
|
|
506
|
+
raw_target,
|
|
507
|
+
exit_code,
|
|
508
|
+
text_error,
|
|
509
|
+
timeout,
|
|
510
|
+
tool_error,
|
|
511
|
+
error_signature,
|
|
512
|
+
observation,
|
|
513
|
+
) = row
|
|
514
|
+
observation = observation or ""
|
|
515
|
+
outcome = _event_outcome(
|
|
516
|
+
action_type, observation, exit_code, text_error, timeout, tool_error
|
|
517
|
+
)
|
|
518
|
+
if outcome == "err" and not error_signature:
|
|
519
|
+
error_signature = "err:unknown"
|
|
520
|
+
events.append(
|
|
521
|
+
(
|
|
522
|
+
event_id,
|
|
523
|
+
session_id,
|
|
524
|
+
event_repo,
|
|
525
|
+
step_idx,
|
|
526
|
+
tool_name,
|
|
527
|
+
action_type,
|
|
528
|
+
verb,
|
|
529
|
+
object_id,
|
|
530
|
+
target,
|
|
531
|
+
raw_target,
|
|
532
|
+
outcome,
|
|
533
|
+
exit_code,
|
|
534
|
+
error_signature,
|
|
535
|
+
outcome == "err",
|
|
536
|
+
(len(observation) + 3) // 4,
|
|
537
|
+
observation[:400],
|
|
538
|
+
)
|
|
539
|
+
)
|
|
540
|
+
con.executemany(
|
|
541
|
+
"""
|
|
542
|
+
INSERT INTO tool_events (
|
|
543
|
+
event_id, session_id, repo, step_idx, tool_name, action_type,
|
|
544
|
+
verb, object_id, target, raw_target, outcome, exit_code,
|
|
545
|
+
error_signature, is_error, approx_out_tokens, obs_excerpt
|
|
546
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
547
|
+
""",
|
|
548
|
+
events,
|
|
549
|
+
)
|
|
550
|
+
session_count = con.execute("SELECT count(*) FROM new_sessions").fetchone()[0]
|
|
551
|
+
event_count = con.execute("SELECT count(*) FROM raw_calls").fetchone()[0]
|
|
552
|
+
stats.append(IngestStats(prefix, matched, session_count, event_count))
|
|
553
|
+
con.execute("DELETE FROM raw_calls")
|
|
554
|
+
con.execute("DELETE FROM new_sessions")
|
|
555
|
+
return stats
|
|
556
|
+
finally:
|
|
557
|
+
con.close()
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def write_sessions(
|
|
561
|
+
db_path: str | Path,
|
|
562
|
+
sessions: Iterable[Session],
|
|
563
|
+
) -> IngestStats:
|
|
564
|
+
"""Write canonical sessions/events, replacing rows with the same ids."""
|
|
565
|
+
session_list = list(sessions)
|
|
566
|
+
con = connect(db_path)
|
|
567
|
+
event_count = 0
|
|
568
|
+
try:
|
|
569
|
+
for session in session_list:
|
|
570
|
+
con.execute("DELETE FROM tool_events WHERE session_id = ?", [session.session_id])
|
|
571
|
+
con.execute("DELETE FROM sessions WHERE session_id = ?", [session.session_id])
|
|
572
|
+
total_turns = len(session.calls)
|
|
573
|
+
con.execute(
|
|
574
|
+
"""
|
|
575
|
+
INSERT INTO sessions (
|
|
576
|
+
session_id, repo, instance_id, resolved, total_turns,
|
|
577
|
+
source, agent, success_score
|
|
578
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
579
|
+
""",
|
|
580
|
+
[
|
|
581
|
+
session.session_id,
|
|
582
|
+
session.repo or "unknown",
|
|
583
|
+
session.instance_id or session.session_id,
|
|
584
|
+
session.resolved,
|
|
585
|
+
total_turns,
|
|
586
|
+
session.source,
|
|
587
|
+
session.agent,
|
|
588
|
+
session.success_score,
|
|
589
|
+
],
|
|
590
|
+
)
|
|
591
|
+
events = []
|
|
592
|
+
for step_idx, call in enumerate(session.calls):
|
|
593
|
+
if call.action_type:
|
|
594
|
+
action = call.action_type
|
|
595
|
+
target = call.target or call.raw_target
|
|
596
|
+
verb = call.verb or call.tool_name
|
|
597
|
+
object_id = call.object_id or target
|
|
598
|
+
else:
|
|
599
|
+
action, target, verb, object_id = _classify(
|
|
600
|
+
call.tool_name, call.raw_target, ""
|
|
601
|
+
)
|
|
602
|
+
observation = call.observation or ""
|
|
603
|
+
exit_code = (
|
|
604
|
+
call.exit_code
|
|
605
|
+
if call.exit_code is not None
|
|
606
|
+
else _extract_exit_code(observation)
|
|
607
|
+
)
|
|
608
|
+
text_error = bool(TEXT_ERROR_RE.search(observation))
|
|
609
|
+
timeout = action == "exec" and bool(TIMEOUT_RE.search(observation))
|
|
610
|
+
tool_error = (
|
|
611
|
+
len(observation) < 500 and bool(TOOL_ERROR_RE.search(observation))
|
|
612
|
+
)
|
|
613
|
+
error_signature = _error_signature(
|
|
614
|
+
observation, exit_code, timeout, text_error, tool_error
|
|
615
|
+
)
|
|
616
|
+
outcome = _event_outcome(
|
|
617
|
+
action, observation, exit_code, text_error, timeout, tool_error
|
|
618
|
+
)
|
|
619
|
+
if call.is_error is True:
|
|
620
|
+
outcome = "err"
|
|
621
|
+
elif call.is_error is False and outcome == "unknown":
|
|
622
|
+
outcome = "ok"
|
|
623
|
+
if outcome == "err" and not error_signature:
|
|
624
|
+
error_signature = "err:unknown"
|
|
625
|
+
events.append(
|
|
626
|
+
(
|
|
627
|
+
f"{session.session_id}:{step_idx}",
|
|
628
|
+
session.session_id,
|
|
629
|
+
session.repo or "unknown",
|
|
630
|
+
step_idx,
|
|
631
|
+
call.segment_idx,
|
|
632
|
+
call.tool_name,
|
|
633
|
+
action,
|
|
634
|
+
verb,
|
|
635
|
+
object_id,
|
|
636
|
+
target,
|
|
637
|
+
call.raw_target,
|
|
638
|
+
outcome,
|
|
639
|
+
exit_code,
|
|
640
|
+
error_signature,
|
|
641
|
+
outcome == "err",
|
|
642
|
+
(len(observation) + 3) // 4,
|
|
643
|
+
observation[:400],
|
|
644
|
+
)
|
|
645
|
+
)
|
|
646
|
+
if events:
|
|
647
|
+
con.executemany(
|
|
648
|
+
"""
|
|
649
|
+
INSERT INTO tool_events (
|
|
650
|
+
event_id, session_id, repo, step_idx, segment_idx,
|
|
651
|
+
tool_name, action_type,
|
|
652
|
+
verb, object_id, target, raw_target, outcome, exit_code,
|
|
653
|
+
error_signature, is_error, approx_out_tokens, obs_excerpt
|
|
654
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
655
|
+
""",
|
|
656
|
+
events,
|
|
657
|
+
)
|
|
658
|
+
event_count += len(events)
|
|
659
|
+
return IngestStats(
|
|
660
|
+
prefix="canonical",
|
|
661
|
+
matched=len(session_list),
|
|
662
|
+
sessions=len(session_list),
|
|
663
|
+
events=event_count,
|
|
664
|
+
)
|
|
665
|
+
finally:
|
|
666
|
+
con.close()
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _flush_sessions(con: duckdb.DuckDBPyConnection, sessions: list[dict]) -> None:
|
|
670
|
+
if not sessions:
|
|
671
|
+
return
|
|
672
|
+
con.executemany(
|
|
673
|
+
"INSERT INTO new_sessions VALUES (?, ?, ?, ?, ?)",
|
|
674
|
+
[
|
|
675
|
+
(s["session_id"], s["repo"], s["instance_id"], s["resolved"], s["total_turns"])
|
|
676
|
+
for s in sessions
|
|
677
|
+
],
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _flush_calls(con: duckdb.DuckDBPyConnection, calls: list[dict]) -> None:
|
|
682
|
+
if not calls:
|
|
683
|
+
return
|
|
684
|
+
con.executemany(
|
|
685
|
+
"INSERT INTO raw_calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
686
|
+
[
|
|
687
|
+
(
|
|
688
|
+
c["event_id"],
|
|
689
|
+
c["session_id"],
|
|
690
|
+
c["repo"],
|
|
691
|
+
c["step_idx"],
|
|
692
|
+
c["tool_name"],
|
|
693
|
+
c["raw_target"],
|
|
694
|
+
c["obs"],
|
|
695
|
+
c["exit_code"],
|
|
696
|
+
c["text_error"],
|
|
697
|
+
c["timeout"],
|
|
698
|
+
c["tool_error"],
|
|
699
|
+
c["error_signature"],
|
|
700
|
+
)
|
|
701
|
+
for c in calls
|
|
702
|
+
],
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _build_target_map(con: duckdb.DuckDBPyConnection) -> None:
|
|
707
|
+
con.execute(
|
|
708
|
+
"CREATE OR REPLACE TEMP TABLE target_map (tool_name VARCHAR, raw_target VARCHAR, target VARCHAR, action_type VARCHAR, verb VARCHAR, object_id VARCHAR)"
|
|
709
|
+
)
|
|
710
|
+
pairs = con.execute("SELECT DISTINCT tool_name, raw_target FROM raw_calls").fetchall()
|
|
711
|
+
if not pairs:
|
|
712
|
+
return
|
|
713
|
+
rows = []
|
|
714
|
+
for tool_name, raw_target in pairs:
|
|
715
|
+
action, target, verb, object_id = _classify(tool_name, raw_target, "")
|
|
716
|
+
rows.append((tool_name, raw_target, target, action, verb, object_id))
|
|
717
|
+
con.executemany("INSERT INTO target_map VALUES (?, ?, ?, ?, ?, ?)", rows)
|