trace2rules 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
trace2rules/ingest.py ADDED
@@ -0,0 +1,717 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import re
6
+ import shlex
7
+ from collections.abc import Iterable
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+
11
+ import duckdb
12
+
13
+ from .db import DB_PATH, connect
14
+ from .models import Session, ToolCall
15
+
16
+ SOURCE = "hf://datasets/nebius/SWE-rebench-openhands-trajectories/trajectories.parquet"
17
+ DEFAULT_REPOS = ["django__django", "pytest-dev__pytest"]
18
+
19
+ EXIT_RE = re.compile(
20
+ r"\[(?:the command completed|command finished) with exit code (-?\d+)\]",
21
+ re.IGNORECASE,
22
+ )
23
+ EXIT_FALLBACK_RE = re.compile(r"(?:Exit code|exit code):\s*(-?\d+)")
24
+ TEXT_ERROR_RE = re.compile(r"Traceback|Error:|ModuleNotFoundError|No such file or directory")
25
+ TOOL_ERROR_RE = re.compile(
26
+ r"Invalid\s+`?path`?|File not found|No such file or directory|does not exist|Permission denied",
27
+ re.IGNORECASE,
28
+ )
29
+ TIMEOUT_RE = re.compile(r"timed out|Timeout", re.IGNORECASE)
30
+
31
+ ENV_PREFIX_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=\S*\s+")
32
+ CD_PREFIX_RE = re.compile(r"^cd\s+[^&;]+\s*(?:&&|;)\s*")
33
+ EXPORT_PREFIX_RE = re.compile(r"^export\s+[^&;]+\s*(?:&&|;)\s*")
34
+ EDIT_VERBS = {"apply_patch", "patch"}
35
+ READ_VERBS = {"cat", "head", "tail", "less", "more", "wc", "nl", "ls", "tree", "stat", "file", "du", "sed", "awk"}
36
+ SEARCH_VERBS = {"grep", "rg", "ag", "ack", "fd", "find"}
37
+ EDITOR_READ_COMMANDS = {"view", "show"}
38
+ FLAG_WITH_VALUE = {"-e", "-A", "-B", "-C", "-m", "-g", "-t", "--include", "--exclude", "--glob", "--type", "-name", "-path", "-type"}
39
+
40
+
41
+ @dataclass
42
+ class IngestStats:
43
+ prefix: str
44
+ matched: int
45
+ sessions: int
46
+ events: int
47
+
48
+
49
+ def _is_path_token(token: str) -> bool:
50
+ if token.startswith("-") or token in {"=", "&&", ";", "|"}:
51
+ return False
52
+ if token.startswith(("2>", "1>", "&>")) or any(char in token for char in "<>|&"):
53
+ return False
54
+ if token.startswith(("skill:", "xd:")):
55
+ return False
56
+ if "/" in token:
57
+ return True
58
+ return bool(re.search(r"\.(py|pyi|txt|md|rst|toml|cfg|ini|yaml|yml|json)$", token))
59
+
60
+
61
+ def normalize_path(path: str, instance_id: str = "") -> str:
62
+ path = (path or "").strip().strip("'\"").rstrip(";,)")
63
+ if not path:
64
+ return ""
65
+ if path.startswith(("skill:", "xd:")):
66
+ return ""
67
+ path = path.replace("\\", "/")
68
+ if path.rstrip("/") in ("/workspace", "/testbed", "/repo", "/app"):
69
+ return "."
70
+ for root in ("/workspace/", "/testbed/", "/repo/", "/app/"):
71
+ if path.startswith(root):
72
+ path = path[len(root) :]
73
+ workspace_root = root == "/workspace/"
74
+ break
75
+ else:
76
+ workspace_root = False
77
+ parts = [p for p in path.split("/") if p not in ("", ".")]
78
+ if workspace_root and parts and (len(parts) > 1 or "__" in parts[0]):
79
+ parts = parts[1:]
80
+ elif parts and ("__" in parts[0] or parts[0] == instance_id):
81
+ parts = parts[1:]
82
+ path = "/".join(parts)
83
+ return path or "."
84
+
85
+
86
+ def _strip_shell_prefix(command: str) -> str:
87
+ command = command.strip()
88
+ while True:
89
+ before = command
90
+ command = ENV_PREFIX_RE.sub("", command)
91
+ command = CD_PREFIX_RE.sub("", command)
92
+ command = EXPORT_PREFIX_RE.sub("", command)
93
+ if command == before:
94
+ return command
95
+
96
+
97
+ def _first_verb(command: str) -> tuple[str, list[str]]:
98
+ command = _strip_shell_prefix(command)
99
+ try:
100
+ tokens = shlex.split(command, posix=True)
101
+ except ValueError:
102
+ tokens = command.split()
103
+ if not tokens:
104
+ return "", []
105
+ if tokens[0] == "git" and len(tokens) > 1:
106
+ return f"git {tokens[1]}", tokens[2:]
107
+ return tokens[0], tokens[1:]
108
+
109
+
110
+ def _basename(path: str) -> str:
111
+ path = (path or "").rstrip("/")
112
+ return path.rsplit("/", 1)[-1] if path else ""
113
+
114
+
115
+ def _shell_verb_tokens(command: str) -> tuple[str, list[str]]:
116
+ verb, tokens = _first_verb(command)
117
+ if verb in ("python", "python3"):
118
+ if "pytest" in tokens:
119
+ return "pytest", tokens
120
+ if "-m" in tokens:
121
+ index = tokens.index("-m")
122
+ module = tokens[index + 1] if index + 1 < len(tokens) else ""
123
+ if module in ("pytest", "pip"):
124
+ return module, tokens
125
+ return module or verb, tokens
126
+ if verb.startswith("git "):
127
+ return "git", [verb.split(" ", 1)[1], *tokens]
128
+ if verb.startswith("pip"):
129
+ return "pip", tokens
130
+ return verb, tokens
131
+
132
+
133
+ def _find_object(tokens: list[str]) -> str:
134
+ path = ""
135
+ name = ""
136
+ index = 0
137
+ while index < len(tokens):
138
+ token = tokens[index]
139
+ if token in ("-name", "-iname") and index + 1 < len(tokens):
140
+ name = tokens[index + 1]
141
+ index += 2
142
+ continue
143
+ if not path and not token.startswith("-"):
144
+ path = token
145
+ index += 1
146
+ if name:
147
+ if path:
148
+ path = normalize_path(path).rstrip("/")
149
+ return f"{path}/{name}" if path not in ("", ".") else name
150
+ return normalize_path(path) if path else "."
151
+
152
+
153
+ def _object_id(
154
+ action: str, verb: str, tokens: list[str], raw_command: str, target: str
155
+ ) -> str:
156
+ if action == "search" and verb == "find":
157
+ return _find_object(tokens)
158
+ if action in ("read", "edit", "search") and " " not in target:
159
+ if not target or target == ".":
160
+ return "" if action == "search" else "."
161
+ return _basename(target) if action in ("read", "edit") else target
162
+
163
+ if verb == "python" and "-c" in tokens:
164
+ payload = raw_command.split("-c", 1)[1].strip() if "-c" in raw_command else ""
165
+ match = re.search(r"\b(?:import|from)\s+([A-Za-z_][\w.]*)", payload)
166
+ if match:
167
+ return match.group(1)
168
+ return "inline:" + hashlib.sha1(payload.encode()).hexdigest()[:8]
169
+
170
+ if verb == "pytest":
171
+ paths = [token for token in tokens if _is_path_token(token)]
172
+ return _basename(paths[-1]) if paths else "pytest"
173
+
174
+ if verb == "python":
175
+ if "-m" in tokens:
176
+ index = tokens.index("-m")
177
+ return tokens[index + 1] if index + 1 < len(tokens) else "python"
178
+ for token in tokens:
179
+ if _is_path_token(token):
180
+ return _basename(token)
181
+ return ""
182
+
183
+ if verb == "pip" and "install" in tokens:
184
+ args = tokens[tokens.index("install") + 1 :]
185
+ if "-e" in args:
186
+ index = args.index("-e")
187
+ destination = args[index + 1] if index + 1 < len(args) else ""
188
+ return "editable" if destination in (".", "./") else _basename(destination)
189
+ packages = [arg for arg in args if not arg.startswith("-")]
190
+ return ",".join(packages[:2]) or "install"
191
+ if verb == "pip":
192
+ return ""
193
+
194
+ if verb == "git":
195
+ return tokens[0] if tokens else ""
196
+ if verb == "export":
197
+ match = re.search(r"\bexport\s+([A-Za-z_][A-Za-z0-9_]*)", raw_command)
198
+ return match.group(1) if match else ""
199
+ if action == "exec" and target and " " not in target:
200
+ return _basename(target)
201
+ return ""
202
+
203
+
204
+ def _extract_exit_code(observation: str) -> int | None:
205
+ match = EXIT_RE.search(observation) or EXIT_FALLBACK_RE.search(observation)
206
+ return int(match.group(1)) if match else None
207
+
208
+
209
+ def _error_signature(
210
+ observation: str,
211
+ exit_code: int | None,
212
+ timeout: bool,
213
+ text_error: bool,
214
+ tool_error: bool,
215
+ ) -> str:
216
+ if timeout:
217
+ return "timeout"
218
+ lowered = observation.lower()
219
+ if "did not appear verbatim" in lowered:
220
+ return "editor:verbatim"
221
+ if "invalid `path`" in lowered or "invalid path" in lowered:
222
+ return "editor:invalid_path"
223
+ matches = re.findall(
224
+ r"(?:^|\n)\s*(?:E\s+)?([A-Za-z_][A-Za-z0-9_.]*(?:Error|Exception))(?::|\s)",
225
+ observation,
226
+ )
227
+ if matches:
228
+ return f"exception:{matches[-1]}"
229
+ if exit_code not in (None, 0):
230
+ return f"exit:{exit_code}"
231
+ if text_error:
232
+ return "text_error"
233
+ if tool_error:
234
+ return "tool_error"
235
+ return ""
236
+
237
+
238
+ def _event_outcome(
239
+ action: str,
240
+ observation: str,
241
+ exit_code: int | None,
242
+ text_error: bool,
243
+ timeout: bool,
244
+ tool_error: bool,
245
+ ) -> str:
246
+ if exit_code is not None:
247
+ return "ok" if exit_code == 0 else "err"
248
+ if timeout:
249
+ return "timeout"
250
+ if action == "edit":
251
+ if tool_error or text_error:
252
+ return "err"
253
+ return "ok" if observation else "unknown"
254
+ if action == "exec" and text_error:
255
+ return "err"
256
+ if action in ("read", "search"):
257
+ if tool_error:
258
+ return "err"
259
+ return "ok" if observation else "unknown"
260
+ return "unknown"
261
+
262
+
263
+ def _path_args(verb: str, tokens: list[str]) -> list[str]:
264
+ if verb == "find":
265
+ for token in tokens:
266
+ if not token.startswith("-"):
267
+ return [token]
268
+ return []
269
+ out: list[str] = []
270
+ skip_next = False
271
+ pattern_token: str | None = None
272
+ for token in tokens:
273
+ if skip_next:
274
+ skip_next = False
275
+ continue
276
+ if token in FLAG_WITH_VALUE:
277
+ skip_next = True
278
+ continue
279
+ if token.startswith("-"):
280
+ continue
281
+ if verb in {"grep", "rg", "ag", "ack", "git grep"} and pattern_token is None:
282
+ pattern_token = token
283
+ continue
284
+ if _is_path_token(token):
285
+ out.append(token)
286
+ return out
287
+
288
+
289
+ def _classify_shell(command: str, instance_id: str) -> tuple[str, str]:
290
+ verb, tokens = _first_verb(command)
291
+ lower = command.lower()
292
+ if verb in EDIT_VERBS or lower.startswith("git apply") or re.search(r"\bsed\s+-i|\bperl\s+-[a-z]*i", lower):
293
+ paths = _path_args(verb, tokens)
294
+ return "edit", normalize_path(paths[-1], instance_id) if paths else _parameterize(command, instance_id)
295
+ if verb in SEARCH_VERBS:
296
+ paths = _path_args(verb, tokens)
297
+ return "search", normalize_path(paths[0], instance_id) if paths else ""
298
+ if verb in READ_VERBS:
299
+ paths = _path_args(verb, tokens)
300
+ return "read", normalize_path(paths[0], instance_id) if paths else "."
301
+ if verb in {"git diff", "git log", "git status", "git show", "git blame"}:
302
+ return "read", verb
303
+ return "exec", _parameterize(command, instance_id)
304
+
305
+
306
+ def _parameterize(command: str, instance_id: str) -> str:
307
+ command = _strip_shell_prefix(" ".join((command or "").split()))
308
+ if not command:
309
+ return ""
310
+ try:
311
+ tokens = shlex.split(command, posix=True)
312
+ except ValueError:
313
+ tokens = command.split()
314
+
315
+ out: list[str] = []
316
+ i = 0
317
+ while i < len(tokens):
318
+ token = tokens[i]
319
+ if token == "-k" and i + 1 < len(tokens):
320
+ out.extend([token, "<arg>"])
321
+ i += 2
322
+ continue
323
+ if _is_path_token(token):
324
+ out.append("<path>")
325
+ else:
326
+ out.append(token)
327
+ i += 1
328
+ result = " ".join(out)
329
+ return result[:240]
330
+
331
+
332
+ def _classify(tool_name: str, raw_target: str, instance_id: str) -> tuple[str, str, str, str]:
333
+ if tool_name == "str_replace_editor":
334
+ command = raw_target.split("\x00", 1)[0]
335
+ path = raw_target.split("\x00", 1)[1] if "\x00" in raw_target else ""
336
+ action = "read" if command in EDITOR_READ_COMMANDS else "edit"
337
+ target = normalize_path(path, instance_id)
338
+ return action, target, action, _basename(target) if target != "." else "."
339
+ action, target = _classify_shell(raw_target, instance_id)
340
+ verb, tokens = _shell_verb_tokens(raw_target)
341
+ object_id = _object_id(action, verb, tokens, raw_target, target)
342
+ return action, target, verb, object_id
343
+
344
+
345
+ def _tool_call_parts(tool_name: str, arguments: str) -> tuple[str, str]:
346
+ try:
347
+ args = json.loads(arguments or "{}")
348
+ except json.JSONDecodeError:
349
+ return tool_name, ""
350
+ if tool_name == "str_replace_editor":
351
+ command = str(args.get("command") or args.get("action") or "")
352
+ path = str(args.get("path") or args.get("file") or "")
353
+ return f"{command}\x00{path}", path or command
354
+ command = str(args.get("command") or args.get("cmd") or "")
355
+ return command, command
356
+
357
+
358
+ def _observation_for_call(call_id: str, obs_by_id: dict[str, str], rows: list[dict], index: int) -> str:
359
+ if call_id and call_id in obs_by_id:
360
+ return obs_by_id[call_id]
361
+ for msg in rows[index + 1 :]:
362
+ if msg["role"] == "tool":
363
+ return msg["content"] or ""
364
+ if msg["role"] == "assistant":
365
+ break
366
+ return ""
367
+
368
+
369
+ def _parse_row(row: tuple) -> tuple[dict, list[dict]]:
370
+ trajectory_id, instance_id, repo, resolved, trajectory = row
371
+ messages = list(trajectory or [])
372
+ obs_by_id = {
373
+ m["tool_call_id"]: (m["content"] or "") for m in messages if m["role"] == "tool"
374
+ }
375
+ calls: list[dict] = []
376
+ step = 0
377
+ for index, msg in enumerate(messages):
378
+ for tool_call in msg.get("tool_calls") or []:
379
+ fn = tool_call["function"]
380
+ raw_target, _ = _tool_call_parts(fn["name"], fn.get("arguments", ""))
381
+ observation = _observation_for_call(tool_call.get("id", ""), obs_by_id, messages, index)
382
+ exit_code = _extract_exit_code(observation)
383
+ text_error = bool(TEXT_ERROR_RE.search(observation))
384
+ timeout = bool(TIMEOUT_RE.search(observation))
385
+ tool_error = len(observation) < 500 and bool(TOOL_ERROR_RE.search(observation))
386
+ calls.append(
387
+ {
388
+ "event_id": f"{trajectory_id}:{step}",
389
+ "session_id": trajectory_id,
390
+ "repo": repo,
391
+ "step_idx": step,
392
+ "tool_name": fn["name"],
393
+ "raw_target": raw_target,
394
+ "obs": observation,
395
+ "exit_code": exit_code,
396
+ "text_error": text_error,
397
+ "timeout": timeout,
398
+ "tool_error": tool_error,
399
+ "error_signature": _error_signature(
400
+ observation, exit_code, timeout, text_error, tool_error
401
+ ),
402
+ }
403
+ )
404
+ step += 1
405
+ session = {
406
+ "session_id": trajectory_id,
407
+ "repo": repo,
408
+ "instance_id": instance_id,
409
+ "resolved": bool(resolved),
410
+ "total_turns": step,
411
+ }
412
+ return session, calls
413
+
414
+
415
+ def _stream_rows(source: str, prefix: str, limit: int):
416
+ reader = duckdb.connect(
417
+ config={
418
+ "memory_limit": "4500MB",
419
+ "threads": "1",
420
+ "preserve_insertion_order": "false",
421
+ }
422
+ )
423
+ try:
424
+ if source.startswith(("hf://", "http://", "https://")):
425
+ try:
426
+ reader.execute("LOAD httpfs")
427
+ except duckdb.Error:
428
+ reader.execute("INSTALL httpfs")
429
+ reader.execute("LOAD httpfs")
430
+ limit_sql = f"LIMIT {int(limit)}" if limit else ""
431
+ query = f"""
432
+ SELECT trajectory_id, instance_id, repo, resolved, trajectory
433
+ FROM read_parquet(?)
434
+ WHERE starts_with(instance_id, ?)
435
+ {limit_sql}
436
+ """
437
+ cursor = reader.execute(query, [source, prefix])
438
+ while True:
439
+ batch = cursor.fetchmany(16)
440
+ if not batch:
441
+ return
442
+ yield from batch
443
+ finally:
444
+ reader.close()
445
+
446
+
447
+ def ingest(
448
+ repos: list[str] | None = None,
449
+ limit: int = 200,
450
+ db_path: str | Path = DB_PATH,
451
+ source: str = SOURCE,
452
+ ) -> list[IngestStats]:
453
+ repos = repos or DEFAULT_REPOS
454
+ con = connect(db_path)
455
+ con.execute("CREATE OR REPLACE TEMP TABLE new_sessions (session_id VARCHAR, repo VARCHAR, instance_id VARCHAR, resolved BOOLEAN, total_turns INTEGER)")
456
+ con.execute(
457
+ "CREATE OR REPLACE TEMP TABLE raw_calls (event_id VARCHAR, session_id VARCHAR, repo VARCHAR, step_idx INTEGER, tool_name VARCHAR, raw_target VARCHAR, obs_content VARCHAR, exit_code INTEGER, text_error BOOLEAN, timeout BOOLEAN, tool_error BOOLEAN, error_signature VARCHAR)"
458
+ )
459
+ stats: list[IngestStats] = []
460
+ try:
461
+ for prefix in repos:
462
+ matched = 0
463
+ rows_for_prefix: list[dict] = []
464
+ new_sessions_batch: list[dict] = []
465
+ for row in _stream_rows(source, prefix, limit):
466
+ matched += 1
467
+ session, calls = _parse_row(row)
468
+ new_sessions_batch.append(session)
469
+ rows_for_prefix.extend(calls)
470
+ if len(rows_for_prefix) >= 2_000:
471
+ _flush_calls(con, rows_for_prefix)
472
+ rows_for_prefix.clear()
473
+ if len(new_sessions_batch) >= 64:
474
+ _flush_sessions(con, new_sessions_batch)
475
+ new_sessions_batch.clear()
476
+ _flush_calls(con, rows_for_prefix)
477
+ _flush_sessions(con, new_sessions_batch)
478
+ con.execute("DELETE FROM tool_events WHERE session_id IN (SELECT session_id FROM new_sessions)")
479
+ con.execute("DELETE FROM sessions WHERE session_id IN (SELECT session_id FROM new_sessions)")
480
+ con.execute(
481
+ "INSERT INTO sessions (session_id, repo, instance_id, resolved, total_turns) "
482
+ "SELECT session_id, repo, instance_id, resolved, total_turns FROM new_sessions"
483
+ )
484
+ _build_target_map(con)
485
+ rows = con.execute(
486
+ """
487
+ SELECT r.event_id, r.session_id, r.repo, r.step_idx, r.tool_name,
488
+ m.action_type, m.verb, m.object_id, m.target, r.raw_target,
489
+ r.exit_code, r.text_error, r.timeout, r.tool_error,
490
+ r.error_signature, r.obs_content
491
+ FROM raw_calls r JOIN target_map m USING (tool_name, raw_target)
492
+ """
493
+ ).fetchall()
494
+ events = []
495
+ for row in rows:
496
+ (
497
+ event_id,
498
+ session_id,
499
+ event_repo,
500
+ step_idx,
501
+ tool_name,
502
+ action_type,
503
+ verb,
504
+ object_id,
505
+ target,
506
+ raw_target,
507
+ exit_code,
508
+ text_error,
509
+ timeout,
510
+ tool_error,
511
+ error_signature,
512
+ observation,
513
+ ) = row
514
+ observation = observation or ""
515
+ outcome = _event_outcome(
516
+ action_type, observation, exit_code, text_error, timeout, tool_error
517
+ )
518
+ if outcome == "err" and not error_signature:
519
+ error_signature = "err:unknown"
520
+ events.append(
521
+ (
522
+ event_id,
523
+ session_id,
524
+ event_repo,
525
+ step_idx,
526
+ tool_name,
527
+ action_type,
528
+ verb,
529
+ object_id,
530
+ target,
531
+ raw_target,
532
+ outcome,
533
+ exit_code,
534
+ error_signature,
535
+ outcome == "err",
536
+ (len(observation) + 3) // 4,
537
+ observation[:400],
538
+ )
539
+ )
540
+ con.executemany(
541
+ """
542
+ INSERT INTO tool_events (
543
+ event_id, session_id, repo, step_idx, tool_name, action_type,
544
+ verb, object_id, target, raw_target, outcome, exit_code,
545
+ error_signature, is_error, approx_out_tokens, obs_excerpt
546
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
547
+ """,
548
+ events,
549
+ )
550
+ session_count = con.execute("SELECT count(*) FROM new_sessions").fetchone()[0]
551
+ event_count = con.execute("SELECT count(*) FROM raw_calls").fetchone()[0]
552
+ stats.append(IngestStats(prefix, matched, session_count, event_count))
553
+ con.execute("DELETE FROM raw_calls")
554
+ con.execute("DELETE FROM new_sessions")
555
+ return stats
556
+ finally:
557
+ con.close()
558
+
559
+
560
+ def write_sessions(
561
+ db_path: str | Path,
562
+ sessions: Iterable[Session],
563
+ ) -> IngestStats:
564
+ """Write canonical sessions/events, replacing rows with the same ids."""
565
+ session_list = list(sessions)
566
+ con = connect(db_path)
567
+ event_count = 0
568
+ try:
569
+ for session in session_list:
570
+ con.execute("DELETE FROM tool_events WHERE session_id = ?", [session.session_id])
571
+ con.execute("DELETE FROM sessions WHERE session_id = ?", [session.session_id])
572
+ total_turns = len(session.calls)
573
+ con.execute(
574
+ """
575
+ INSERT INTO sessions (
576
+ session_id, repo, instance_id, resolved, total_turns,
577
+ source, agent, success_score
578
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
579
+ """,
580
+ [
581
+ session.session_id,
582
+ session.repo or "unknown",
583
+ session.instance_id or session.session_id,
584
+ session.resolved,
585
+ total_turns,
586
+ session.source,
587
+ session.agent,
588
+ session.success_score,
589
+ ],
590
+ )
591
+ events = []
592
+ for step_idx, call in enumerate(session.calls):
593
+ if call.action_type:
594
+ action = call.action_type
595
+ target = call.target or call.raw_target
596
+ verb = call.verb or call.tool_name
597
+ object_id = call.object_id or target
598
+ else:
599
+ action, target, verb, object_id = _classify(
600
+ call.tool_name, call.raw_target, ""
601
+ )
602
+ observation = call.observation or ""
603
+ exit_code = (
604
+ call.exit_code
605
+ if call.exit_code is not None
606
+ else _extract_exit_code(observation)
607
+ )
608
+ text_error = bool(TEXT_ERROR_RE.search(observation))
609
+ timeout = action == "exec" and bool(TIMEOUT_RE.search(observation))
610
+ tool_error = (
611
+ len(observation) < 500 and bool(TOOL_ERROR_RE.search(observation))
612
+ )
613
+ error_signature = _error_signature(
614
+ observation, exit_code, timeout, text_error, tool_error
615
+ )
616
+ outcome = _event_outcome(
617
+ action, observation, exit_code, text_error, timeout, tool_error
618
+ )
619
+ if call.is_error is True:
620
+ outcome = "err"
621
+ elif call.is_error is False and outcome == "unknown":
622
+ outcome = "ok"
623
+ if outcome == "err" and not error_signature:
624
+ error_signature = "err:unknown"
625
+ events.append(
626
+ (
627
+ f"{session.session_id}:{step_idx}",
628
+ session.session_id,
629
+ session.repo or "unknown",
630
+ step_idx,
631
+ call.segment_idx,
632
+ call.tool_name,
633
+ action,
634
+ verb,
635
+ object_id,
636
+ target,
637
+ call.raw_target,
638
+ outcome,
639
+ exit_code,
640
+ error_signature,
641
+ outcome == "err",
642
+ (len(observation) + 3) // 4,
643
+ observation[:400],
644
+ )
645
+ )
646
+ if events:
647
+ con.executemany(
648
+ """
649
+ INSERT INTO tool_events (
650
+ event_id, session_id, repo, step_idx, segment_idx,
651
+ tool_name, action_type,
652
+ verb, object_id, target, raw_target, outcome, exit_code,
653
+ error_signature, is_error, approx_out_tokens, obs_excerpt
654
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
655
+ """,
656
+ events,
657
+ )
658
+ event_count += len(events)
659
+ return IngestStats(
660
+ prefix="canonical",
661
+ matched=len(session_list),
662
+ sessions=len(session_list),
663
+ events=event_count,
664
+ )
665
+ finally:
666
+ con.close()
667
+
668
+
669
+ def _flush_sessions(con: duckdb.DuckDBPyConnection, sessions: list[dict]) -> None:
670
+ if not sessions:
671
+ return
672
+ con.executemany(
673
+ "INSERT INTO new_sessions VALUES (?, ?, ?, ?, ?)",
674
+ [
675
+ (s["session_id"], s["repo"], s["instance_id"], s["resolved"], s["total_turns"])
676
+ for s in sessions
677
+ ],
678
+ )
679
+
680
+
681
+ def _flush_calls(con: duckdb.DuckDBPyConnection, calls: list[dict]) -> None:
682
+ if not calls:
683
+ return
684
+ con.executemany(
685
+ "INSERT INTO raw_calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
686
+ [
687
+ (
688
+ c["event_id"],
689
+ c["session_id"],
690
+ c["repo"],
691
+ c["step_idx"],
692
+ c["tool_name"],
693
+ c["raw_target"],
694
+ c["obs"],
695
+ c["exit_code"],
696
+ c["text_error"],
697
+ c["timeout"],
698
+ c["tool_error"],
699
+ c["error_signature"],
700
+ )
701
+ for c in calls
702
+ ],
703
+ )
704
+
705
+
706
+ def _build_target_map(con: duckdb.DuckDBPyConnection) -> None:
707
+ con.execute(
708
+ "CREATE OR REPLACE TEMP TABLE target_map (tool_name VARCHAR, raw_target VARCHAR, target VARCHAR, action_type VARCHAR, verb VARCHAR, object_id VARCHAR)"
709
+ )
710
+ pairs = con.execute("SELECT DISTINCT tool_name, raw_target FROM raw_calls").fetchall()
711
+ if not pairs:
712
+ return
713
+ rows = []
714
+ for tool_name, raw_target in pairs:
715
+ action, target, verb, object_id = _classify(tool_name, raw_target, "")
716
+ rows.append((tool_name, raw_target, target, action, verb, object_id))
717
+ con.executemany("INSERT INTO target_map VALUES (?, ?, ?, ?, ?, ?)", rows)