trace2rules 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trace2rules-0.1.3 → trace2rules-0.1.4}/PKG-INFO +1 -1
- {trace2rules-0.1.3 → trace2rules-0.1.4}/pyproject.toml +1 -1
- {trace2rules-0.1.3 → trace2rules-0.1.4}/pyproject.toml.orig +1 -1
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/cli.py +22 -2
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/cluster.py +2 -1
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/db.py +9 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/detectors.py +34 -0
- trace2rules-0.1.4/src/trace2rules/feedback.py +108 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/ingest.py +24 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/models.py +9 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/sources/agent_trajectory.py +28 -3
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/synthesize.py +7 -3
- {trace2rules-0.1.3 → trace2rules-0.1.4}/LICENSE +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/README.md +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/__init__.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/agents.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/evaluate.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/labels.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/patterns.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/reconcile.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/redact.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/sources/__init__.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/sources/common.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/sources/opencode.py +0 -0
- {trace2rules-0.1.3 → trace2rules-0.1.4}/src/trace2rules/sources/swe_chat.py +0 -0
|
@@ -282,6 +282,24 @@ def _print_detector_tables(mined: dict, limit: int = 10) -> None:
|
|
|
282
282
|
)
|
|
283
283
|
console.print(table)
|
|
284
284
|
|
|
285
|
+
feedback = mined.get("feedback", [])[:limit]
|
|
286
|
+
if feedback:
|
|
287
|
+
table = Table(title="Repeated human steering", header_style="bold", expand=True)
|
|
288
|
+
table.add_column("Repo")
|
|
289
|
+
table.add_column("Constraint")
|
|
290
|
+
table.add_column("Mentions", justify="right")
|
|
291
|
+
table.add_column("Sessions", justify="right")
|
|
292
|
+
table.add_column("Example")
|
|
293
|
+
for finding in feedback:
|
|
294
|
+
table.add_row(
|
|
295
|
+
escape(finding.repo),
|
|
296
|
+
escape(finding.title),
|
|
297
|
+
str(finding.tokens),
|
|
298
|
+
str(finding.sessions),
|
|
299
|
+
escape(finding.cause or finding.detail),
|
|
300
|
+
)
|
|
301
|
+
console.print(table)
|
|
302
|
+
|
|
285
303
|
blasts = mined["context"][:limit]
|
|
286
304
|
if blasts:
|
|
287
305
|
table = Table(title="Context-blasting calls", header_style="bold", expand=True)
|
|
@@ -365,7 +383,8 @@ def _mine_command(
|
|
|
365
383
|
console.print(table)
|
|
366
384
|
console.print(
|
|
367
385
|
f"[dim]{len(mined['hotspots'])} hotspots, {len(mined['recovery'])} recovery loops, "
|
|
368
|
-
f"{len(mined['context'])} context blasts, {len(mined.get('orientation', []))} orientation
|
|
386
|
+
f"{len(mined['context'])} context blasts, {len(mined.get('orientation', []))} orientation, "
|
|
387
|
+
f"{len(mined.get('feedback', []))} repeated steering[/dim]"
|
|
369
388
|
)
|
|
370
389
|
finally:
|
|
371
390
|
con.close()
|
|
@@ -436,7 +455,7 @@ def _report_command(
|
|
|
436
455
|
mined = mine_friction(con, top=None if (repo or prompt) else top)
|
|
437
456
|
findings = [
|
|
438
457
|
finding
|
|
439
|
-
for key in ("hotspots", "recovery", "context", "orientation", "selector_miss", "workaround", "stuck_loop", "noop_retry", "premature_completion", "low_validation", "site_edit_mismatch")
|
|
458
|
+
for key in ("hotspots", "recovery", "context", "orientation", "selector_miss", "workaround", "stuck_loop", "noop_retry", "premature_completion", "low_validation", "site_edit_mismatch", "feedback")
|
|
440
459
|
for finding in mined[key]
|
|
441
460
|
]
|
|
442
461
|
exact = []
|
|
@@ -944,6 +963,7 @@ def _exact_command(
|
|
|
944
963
|
app.command("exact")(_exact_command)
|
|
945
964
|
|
|
946
965
|
|
|
966
|
+
|
|
947
967
|
def _synthesize_command(
|
|
948
968
|
repo: Annotated[str | None, typer.Option("--repo", help="Synthesize only this repo.")] = None,
|
|
949
969
|
top: Annotated[int, typer.Option(help="Findings to pass to synthesis.")] = 25,
|
|
@@ -8,7 +8,7 @@ from sklearn.cluster import AgglomerativeClustering
|
|
|
8
8
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
9
9
|
from sklearn.metrics.pairwise import cosine_distances
|
|
10
10
|
|
|
11
|
-
from .detectors import Finding, context_blasts, global_patterns, hotspots, noop_retries, orientation, premature_completion, low_validation, site_edit_mismatch, recovery_loops, selector_misses, stuck_loops, workaround_scripts
|
|
11
|
+
from .detectors import Finding, context_blasts, feedback_loops, global_patterns, hotspots, noop_retries, orientation, premature_completion, low_validation, site_edit_mismatch, recovery_loops, selector_misses, stuck_loops, workaround_scripts
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
@dataclass
|
|
@@ -353,5 +353,6 @@ def mine(
|
|
|
353
353
|
"premature_completion": premature_completion(con, limit=top),
|
|
354
354
|
"low_validation": low_validation(con, limit=top),
|
|
355
355
|
"site_edit_mismatch": site_edit_mismatch(con, limit=top),
|
|
356
|
+
"feedback": feedback_loops(con, limit=top),
|
|
356
357
|
"clusters": [],
|
|
357
358
|
}
|
|
@@ -55,6 +55,15 @@ CREATE TABLE IF NOT EXISTS tool_events (
|
|
|
55
55
|
event_hash VARCHAR DEFAULT ''
|
|
56
56
|
);
|
|
57
57
|
|
|
58
|
+
CREATE TABLE IF NOT EXISTS feedback_events (
|
|
59
|
+
session_id VARCHAR NOT NULL,
|
|
60
|
+
step_idx INTEGER NOT NULL,
|
|
61
|
+
kind VARCHAR NOT NULL,
|
|
62
|
+
text VARCHAR NOT NULL,
|
|
63
|
+
keys VARCHAR NOT NULL,
|
|
64
|
+
repo VARCHAR DEFAULT ''
|
|
65
|
+
);
|
|
66
|
+
|
|
58
67
|
CREATE TABLE IF NOT EXISTS target_labels (
|
|
59
68
|
tool_name VARCHAR NOT NULL,
|
|
60
69
|
action_type VARCHAR NOT NULL,
|
|
@@ -882,6 +882,39 @@ def global_patterns(
|
|
|
882
882
|
]
|
|
883
883
|
|
|
884
884
|
|
|
885
|
+
def feedback_loops(con: duckdb.DuckDBPyConnection, limit: int | None = None) -> list[Finding]:
|
|
886
|
+
query = """
|
|
887
|
+
SELECT repo, keys, count(*) AS mentions,
|
|
888
|
+
count(DISTINCT session_id) AS sessions,
|
|
889
|
+
arg_max(text, step_idx) AS example,
|
|
890
|
+
list(DISTINCT substr(text, 1, 300)) AS examples
|
|
891
|
+
FROM feedback_events
|
|
892
|
+
WHERE keys <> ''
|
|
893
|
+
GROUP BY 1, 2
|
|
894
|
+
HAVING count(*) >= 2
|
|
895
|
+
ORDER BY sessions DESC, mentions DESC
|
|
896
|
+
"""
|
|
897
|
+
if limit is not None:
|
|
898
|
+
query += f" LIMIT {int(limit)}"
|
|
899
|
+
return [
|
|
900
|
+
Finding(
|
|
901
|
+
kind="feedback_loop",
|
|
902
|
+
repo=repo,
|
|
903
|
+
title=key,
|
|
904
|
+
sessions=sessions,
|
|
905
|
+
tokens=mentions,
|
|
906
|
+
detail=f"{mentions} mentions across {sessions} sessions",
|
|
907
|
+
side="agent",
|
|
908
|
+
recommendation=(
|
|
909
|
+
f"The user repeated the `{key}` constraint. Treat it as blocking: "
|
|
910
|
+
"restate it, inspect the current setting, and change the next action instead of retrying."
|
|
911
|
+
),
|
|
912
|
+
cause=(example or "")[:240],
|
|
913
|
+
)
|
|
914
|
+
for repo, key, mentions, sessions, example, examples in con.execute(query).fetchall()
|
|
915
|
+
]
|
|
916
|
+
|
|
917
|
+
|
|
885
918
|
def all_findings(con: duckdb.DuckDBPyConnection, limit: int = 25) -> list[Finding]:
|
|
886
919
|
findings: list[Finding] = []
|
|
887
920
|
findings.extend(hotspots(con, limit=limit))
|
|
@@ -896,4 +929,5 @@ def all_findings(con: duckdb.DuckDBPyConnection, limit: int = 25) -> list[Findin
|
|
|
896
929
|
findings.extend(premature_completion(con, limit=limit))
|
|
897
930
|
findings.extend(low_validation(con, limit=limit))
|
|
898
931
|
findings.extend(site_edit_mismatch(con, limit=limit))
|
|
932
|
+
findings.extend(feedback_loops(con, limit=limit))
|
|
899
933
|
return findings
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
TOPICS = (
|
|
10
|
+
"cpu", "memory", "ram", "cache", "caching", "schema", "index", "test",
|
|
11
|
+
"benchmark", "load", "wsl", "timeout", "sqlite", "postgres", "build",
|
|
12
|
+
"compile", "server", "docker", "gpu", "port", "config",
|
|
13
|
+
)
|
|
14
|
+
NEGATION = re.compile(
|
|
15
|
+
r"(?i)\b(no|not|never|don't|do not|without|stop|avoid|limit|cap|reduce|lower|only)\b"
|
|
16
|
+
)
|
|
17
|
+
ACCEPTANCE = re.compile(r"(?i)\b(yes|good|great|perfect|thanks|thank you|lgtm|nice|works)\b")
|
|
18
|
+
SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(slots=True)
|
|
22
|
+
class SteeringFinding:
|
|
23
|
+
repo: str
|
|
24
|
+
session: str
|
|
25
|
+
key: str
|
|
26
|
+
occurrences: int
|
|
27
|
+
examples: tuple[str, ...]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def user_turns(path: Path) -> list[str]:
|
|
31
|
+
turns: list[str] = []
|
|
32
|
+
try:
|
|
33
|
+
handle = path.open(encoding="utf-8", errors="replace")
|
|
34
|
+
except OSError:
|
|
35
|
+
return turns
|
|
36
|
+
with handle:
|
|
37
|
+
for line in handle:
|
|
38
|
+
try:
|
|
39
|
+
record = json.loads(line)
|
|
40
|
+
except json.JSONDecodeError:
|
|
41
|
+
continue
|
|
42
|
+
if record.get("type") == "message":
|
|
43
|
+
message = record.get("message") or {}
|
|
44
|
+
role = message.get("role")
|
|
45
|
+
content = message.get("content")
|
|
46
|
+
else:
|
|
47
|
+
role = record.get("role")
|
|
48
|
+
content = record.get("content")
|
|
49
|
+
if role != "user":
|
|
50
|
+
continue
|
|
51
|
+
if isinstance(content, list):
|
|
52
|
+
text = " ".join(
|
|
53
|
+
part.get("text", "")
|
|
54
|
+
for part in content
|
|
55
|
+
if isinstance(part, dict)
|
|
56
|
+
)
|
|
57
|
+
else:
|
|
58
|
+
text = str(content or "")
|
|
59
|
+
text = text.strip()
|
|
60
|
+
if not text or text.startswith(("<system-reminder>", "<user>")):
|
|
61
|
+
continue
|
|
62
|
+
turns.append(text)
|
|
63
|
+
return turns
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def constraint_keys(text: str) -> tuple[str, ...]:
|
|
67
|
+
keys: set[str] = set()
|
|
68
|
+
for sentence in SENTENCE_SPLIT.split(text):
|
|
69
|
+
if not sentence or ACCEPTANCE.search(sentence) and not NEGATION.search(sentence):
|
|
70
|
+
continue
|
|
71
|
+
if not NEGATION.search(sentence):
|
|
72
|
+
continue
|
|
73
|
+
lower = sentence.lower()
|
|
74
|
+
for topic in TOPICS:
|
|
75
|
+
if re.search(rf"\b{re.escape(topic)}\b", lower):
|
|
76
|
+
keys.add(topic)
|
|
77
|
+
return tuple(sorted(keys))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def mine_repeated_steering(source: str | Path) -> list[SteeringFinding]:
|
|
81
|
+
root = Path(source).expanduser()
|
|
82
|
+
findings: list[SteeringFinding] = []
|
|
83
|
+
if root.is_file():
|
|
84
|
+
files = [root]
|
|
85
|
+
elif root.name == "sessions":
|
|
86
|
+
# OMP stores the main transcript one level below each project directory.
|
|
87
|
+
files = sorted(root.glob("*/*.jsonl"))
|
|
88
|
+
else:
|
|
89
|
+
files = sorted(root.rglob("*.jsonl"))
|
|
90
|
+
for path in files:
|
|
91
|
+
turns = user_turns(path)
|
|
92
|
+
by_key: dict[str, list[str]] = defaultdict(list)
|
|
93
|
+
for text in turns:
|
|
94
|
+
for key in constraint_keys(text):
|
|
95
|
+
by_key[key].append(text)
|
|
96
|
+
for key, messages in by_key.items():
|
|
97
|
+
if len(messages) < 2:
|
|
98
|
+
continue
|
|
99
|
+
findings.append(
|
|
100
|
+
SteeringFinding(
|
|
101
|
+
repo=path.parent.name.lstrip("-") or "unknown",
|
|
102
|
+
session=str(path),
|
|
103
|
+
key=key,
|
|
104
|
+
occurrences=len(messages),
|
|
105
|
+
examples=tuple(message[:240] for message in messages[:3]),
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
return findings
|
|
@@ -887,6 +887,10 @@ def write_sessions(
|
|
|
887
887
|
f"DELETE FROM sessions WHERE session_id IN ({placeholders})",
|
|
888
888
|
session_ids,
|
|
889
889
|
)
|
|
890
|
+
con.execute(
|
|
891
|
+
f"DELETE FROM feedback_events WHERE session_id IN ({placeholders})",
|
|
892
|
+
session_ids,
|
|
893
|
+
)
|
|
890
894
|
_insert_rows(
|
|
891
895
|
con,
|
|
892
896
|
"sessions",
|
|
@@ -984,6 +988,26 @@ def write_sessions(
|
|
|
984
988
|
)
|
|
985
989
|
)
|
|
986
990
|
|
|
991
|
+
feedback_rows = [
|
|
992
|
+
(
|
|
993
|
+
session.session_id,
|
|
994
|
+
feedback.step_idx,
|
|
995
|
+
feedback.kind,
|
|
996
|
+
feedback.text[:2000],
|
|
997
|
+
",".join(feedback.keys),
|
|
998
|
+
session.repo or "unknown",
|
|
999
|
+
)
|
|
1000
|
+
for session in session_list
|
|
1001
|
+
for feedback in session.feedback
|
|
1002
|
+
]
|
|
1003
|
+
if feedback_rows:
|
|
1004
|
+
_insert_rows(
|
|
1005
|
+
con,
|
|
1006
|
+
"feedback_events",
|
|
1007
|
+
feedback_rows,
|
|
1008
|
+
columns=["session_id", "step_idx", "kind", "text", "keys", "repo"],
|
|
1009
|
+
)
|
|
1010
|
+
|
|
987
1011
|
if events:
|
|
988
1012
|
_insert_rows(
|
|
989
1013
|
con,
|
|
@@ -21,6 +21,14 @@ class ToolCall:
|
|
|
21
21
|
assistant_context: str = ""
|
|
22
22
|
|
|
23
23
|
|
|
24
|
+
@dataclass(slots=True)
|
|
25
|
+
class Feedback:
|
|
26
|
+
text: str
|
|
27
|
+
step_idx: int
|
|
28
|
+
kind: str = "feedback"
|
|
29
|
+
keys: tuple[str, ...] = ()
|
|
30
|
+
|
|
31
|
+
|
|
24
32
|
@dataclass(slots=True)
|
|
25
33
|
class Session:
|
|
26
34
|
"""A normalized session accepted by the shared DuckDB writer."""
|
|
@@ -35,3 +43,4 @@ class Session:
|
|
|
35
43
|
agent: str = ""
|
|
36
44
|
success_score: float | None = None
|
|
37
45
|
started_at: datetime | None = None
|
|
46
|
+
feedback: list[Feedback] = field(default_factory=list)
|
|
@@ -14,7 +14,8 @@ from urllib.parse import urlparse
|
|
|
14
14
|
|
|
15
15
|
from rich.progress import Progress
|
|
16
16
|
|
|
17
|
-
from ..
|
|
17
|
+
from ..feedback import constraint_keys
|
|
18
|
+
from ..models import Feedback, Session, ToolCall
|
|
18
19
|
from .common import classify_tool_call, parse_arguments, target_from_args
|
|
19
20
|
|
|
20
21
|
SOURCE_DIRS = {
|
|
@@ -416,6 +417,27 @@ def _records_to_session(
|
|
|
416
417
|
pending: dict[str, ToolCall] = {}
|
|
417
418
|
for record in records:
|
|
418
419
|
role = record.get("role")
|
|
420
|
+
if role == "user":
|
|
421
|
+
content = record.get("content")
|
|
422
|
+
if isinstance(content, list):
|
|
423
|
+
text = " ".join(
|
|
424
|
+
part.get("text", "")
|
|
425
|
+
for part in content
|
|
426
|
+
if isinstance(part, dict)
|
|
427
|
+
)
|
|
428
|
+
else:
|
|
429
|
+
text = str(content or "")
|
|
430
|
+
text = text.strip()
|
|
431
|
+
if text and not text.startswith(("<system-reminder>", "<user>")):
|
|
432
|
+
keys = constraint_keys(text)
|
|
433
|
+
session.feedback.append(
|
|
434
|
+
Feedback(
|
|
435
|
+
text=text,
|
|
436
|
+
step_idx=len(session.calls),
|
|
437
|
+
kind="constraint" if keys else "feedback",
|
|
438
|
+
keys=keys,
|
|
439
|
+
)
|
|
440
|
+
)
|
|
419
441
|
if role == "assistant" and record.get("tool_calls"):
|
|
420
442
|
for tool_call in record["tool_calls"]:
|
|
421
443
|
name = tool_call.get("name") or ""
|
|
@@ -568,7 +590,10 @@ def ingest_agent_sessions(
|
|
|
568
590
|
]
|
|
569
591
|
)
|
|
570
592
|
)
|
|
571
|
-
except
|
|
593
|
+
except Exception:
|
|
594
|
+
# A single malformed or oversized transcript can crash the
|
|
595
|
+
# Node runtime. Isolate each file so the whole ingest does
|
|
596
|
+
# not die with it.
|
|
572
597
|
normalized = []
|
|
573
598
|
for path, text in texts:
|
|
574
599
|
try:
|
|
@@ -579,7 +604,7 @@ def ingest_agent_sessions(
|
|
|
579
604
|
)[0]
|
|
580
605
|
]
|
|
581
606
|
)
|
|
582
|
-
except
|
|
607
|
+
except Exception as error:
|
|
583
608
|
print(f"Skipping {path}: {error}")
|
|
584
609
|
normalized.append({"records": [], "diagnostics": []})
|
|
585
610
|
return normalized
|
|
@@ -19,8 +19,10 @@ from .detectors import Finding
|
|
|
19
19
|
from .redact import redact_value
|
|
20
20
|
|
|
21
21
|
MAX_RULES = 5
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
# Evidence and source refs are part of the output; keep enough room to avoid
|
|
23
|
+
# forcing the model to discard supported rules.
|
|
24
|
+
MAX_LINES = 160
|
|
25
|
+
MAX_CHARS = 12000
|
|
24
26
|
EXACT_LENGTHS = (3, 4, 5)
|
|
25
27
|
|
|
26
28
|
GENERATED_VERIFIER_RE = re.compile(
|
|
@@ -92,6 +94,7 @@ def _candidate_score(finding: Finding) -> float:
|
|
|
92
94
|
"context": 0.7,
|
|
93
95
|
"hotspot": 0.8,
|
|
94
96
|
"site_edit_mismatch": 1.1,
|
|
97
|
+
"feedback_loop": 1.3,
|
|
95
98
|
}
|
|
96
99
|
support = max(finding.sessions, 1)
|
|
97
100
|
impact = max(finding.tokens, finding.cost, 1)
|
|
@@ -116,12 +119,13 @@ def select_findings(
|
|
|
116
119
|
"premature_completion",
|
|
117
120
|
"low_validation",
|
|
118
121
|
"site_edit_mismatch",
|
|
122
|
+
"feedback",
|
|
119
123
|
):
|
|
120
124
|
candidates.extend(
|
|
121
125
|
finding
|
|
122
126
|
for finding in mined.get(key, [])
|
|
123
127
|
if finding.repo == repo
|
|
124
|
-
and finding.sessions >= 3
|
|
128
|
+
and (finding.sessions >= 3 or finding.kind == "feedback_loop")
|
|
125
129
|
and not GENERATED_VERIFIER_RE.search(finding.title)
|
|
126
130
|
and (finding.recommendation or finding.kind != "context")
|
|
127
131
|
and (
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|