trace2rules 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trace2rules-0.1.0 → trace2rules-0.1.1}/PKG-INFO +11 -5
- {trace2rules-0.1.0 → trace2rules-0.1.1}/README.md +10 -4
- {trace2rules-0.1.0 → trace2rules-0.1.1}/pyproject.toml +1 -1
- {trace2rules-0.1.0 → trace2rules-0.1.1}/pyproject.toml.orig +1 -1
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/cli.py +196 -34
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/detectors.py +2 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/ingest.py +5 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/labels.py +3 -2
- trace2rules-0.1.1/src/trace2rules/redact.py +67 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/sources/agent_trajectory.py +152 -11
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/synthesize.py +160 -51
- {trace2rules-0.1.0 → trace2rules-0.1.1}/LICENSE +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/__init__.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/cluster.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/db.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/models.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/patterns.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/sources/__init__.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/sources/common.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/sources/opencode.py +0 -0
- {trace2rules-0.1.0 → trace2rules-0.1.1}/src/trace2rules/sources/swe_chat.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: trace2rules
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Mine coding-agent trajectories into evidence-backed AGENTS.md rules
|
|
5
5
|
Keywords: agents,coding-agents,agents.md,trajectories,duckdb
|
|
6
6
|
Author: Rakha Kanz Kautsar
|
|
@@ -76,7 +76,7 @@ Rules are capped at 5 rules, 60 lines, and 2,500 characters per repo.
|
|
|
76
76
|
|
|
77
77
|
## Synthesis
|
|
78
78
|
|
|
79
|
-
Synthesis is LLM-first. `trace2rules synthesize` sends
|
|
79
|
+
Synthesis is LLM-first. `trace2rules synthesize` sends redacted gated findings for each repo to an OpenAI-compatible chat model and asks for structured rules. Common API keys, bearer tokens, URL credentials, query tokens, password assignments, and private keys are redacted before external calls; redaction is best-effort, not a guarantee.
|
|
80
80
|
|
|
81
81
|
Provider selection:
|
|
82
82
|
|
|
@@ -119,7 +119,7 @@ uv sync
|
|
|
119
119
|
|
|
120
120
|
## Quick start: your own sessions
|
|
121
121
|
|
|
122
|
-
Analyze the sessions already on your machine.
|
|
122
|
+
Analyze the sessions already on your machine.
|
|
123
123
|
|
|
124
124
|
```bash
|
|
125
125
|
uv tool install trace2rules
|
|
@@ -128,6 +128,10 @@ trace2rules mine
|
|
|
128
128
|
trace2rules synthesize
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
+
Or hand the workflow to your coding agent:
|
|
132
|
+
|
|
133
|
+
> Read the `trace2rules` README, install it with `uv tool install trace2rules` if it is missing, then ingest my local agent sessions. Use `trace2rules report` to surface recurring friction with evidence and synthesize repo-specific `AGENTS.md` rules from that JSON.
|
|
134
|
+
|
|
131
135
|
`--source` is repeatable. `--source ~` scans all standard local stores: `~/.claude/projects`, `~/.codex/sessions`, `~/.pi/agent/sessions`, and `~/.omp/agent/sessions`. The example imports 50 sessions per source; use `--limit 0` for all.
|
|
132
136
|
|
|
133
137
|
LLM synthesis is the default. Add `--offline` if you do not have an API key.
|
|
@@ -200,6 +204,7 @@ Use `--repo owner/name` to override the repository label for local sessions.
|
|
|
200
204
|
| `ingest` | Load sessions and tool events into DuckDB |
|
|
201
205
|
| `mine` | Show friction clusters, hotspots, recovery loops, and context blasts |
|
|
202
206
|
| `report` | Emit all findings and patterns as evidence-rich JSON for an agent |
|
|
207
|
+
| `doctor` | Check install, Node, credentials, source stores, and database schema |
|
|
203
208
|
| `label` | Optional: classify targets with a decision model (`intent`, `object_type`, confidence) |
|
|
204
209
|
| `patterns` | Mine structural multi-step friction; labels refine when available |
|
|
205
210
|
| `exact` | Show exact recurring sequences with evidence |
|
|
@@ -214,9 +219,10 @@ trace2rules ingest --format swe-rebench # opt into the public dataset
|
|
|
214
219
|
trace2rules ingest --source PATH --limit 200
|
|
215
220
|
trace2rules synthesize
|
|
216
221
|
trace2rules synthesize --offline
|
|
222
|
+
trace2rules synthesize --max-rules 10
|
|
223
|
+
trace2rules synthesize --out-dir ./proposals
|
|
217
224
|
trace2rules synthesize --repo owner/name --out AGENTS.proposed.md
|
|
218
225
|
trace2rules synthesize --repo owner/name --out -
|
|
219
|
-
trace2rules synthesize --out-dir ./proposals
|
|
220
226
|
```
|
|
221
227
|
|
|
222
228
|
`--label` auto-detects an authenticated `wrangler` first, then `GEMINI_API_KEY`/`GOOGLE_API_KEY`, then `OPENAI_API_KEY`. It is opt-in because a full local run can have thousands of unique targets.
|
|
@@ -269,7 +275,7 @@ trace2rules patterns --min-confidence 0.4
|
|
|
269
275
|
|
|
270
276
|
Cloudflare models authenticate through `wrangler auth token --json` and `wrangler whoami --json`; the first account is used. Gemini and OpenAI use their standard keys. Labeling defaults to 270 requests/minute with 6 workers and retries 429/5xx responses.
|
|
271
277
|
|
|
272
|
-
`label` sends the tool name, action type, normalized target, and a raw command example to the selected provider. Do not run
|
|
278
|
+
`label` sends the tool name, action type, redacted normalized target, and a redacted raw command example to the selected provider. The local DuckDB and `report` output remain unredacted. Do not run external labeling or synthesis on private sessions without reviewing the redaction and provider policy.
|
|
273
279
|
|
|
274
280
|
## Configuration
|
|
275
281
|
|
|
@@ -47,7 +47,7 @@ Rules are capped at 5 rules, 60 lines, and 2,500 characters per repo.
|
|
|
47
47
|
|
|
48
48
|
## Synthesis
|
|
49
49
|
|
|
50
|
-
Synthesis is LLM-first. `trace2rules synthesize` sends
|
|
50
|
+
Synthesis is LLM-first. `trace2rules synthesize` sends redacted gated findings for each repo to an OpenAI-compatible chat model and asks for structured rules. Common API keys, bearer tokens, URL credentials, query tokens, password assignments, and private keys are redacted before external calls; redaction is best-effort, not a guarantee.
|
|
51
51
|
|
|
52
52
|
Provider selection:
|
|
53
53
|
|
|
@@ -90,7 +90,7 @@ uv sync
|
|
|
90
90
|
|
|
91
91
|
## Quick start: your own sessions
|
|
92
92
|
|
|
93
|
-
Analyze the sessions already on your machine.
|
|
93
|
+
Analyze the sessions already on your machine.
|
|
94
94
|
|
|
95
95
|
```bash
|
|
96
96
|
uv tool install trace2rules
|
|
@@ -99,6 +99,10 @@ trace2rules mine
|
|
|
99
99
|
trace2rules synthesize
|
|
100
100
|
```
|
|
101
101
|
|
|
102
|
+
Or hand the workflow to your coding agent:
|
|
103
|
+
|
|
104
|
+
> Read the `trace2rules` README, install it with `uv tool install trace2rules` if it is missing, then ingest my local agent sessions. Use `trace2rules report` to surface recurring friction with evidence and synthesize repo-specific `AGENTS.md` rules from that JSON.
|
|
105
|
+
|
|
102
106
|
`--source` is repeatable. `--source ~` scans all standard local stores: `~/.claude/projects`, `~/.codex/sessions`, `~/.pi/agent/sessions`, and `~/.omp/agent/sessions`. The example imports 50 sessions per source; use `--limit 0` for all.
|
|
103
107
|
|
|
104
108
|
LLM synthesis is the default. Add `--offline` if you do not have an API key.
|
|
@@ -171,6 +175,7 @@ Use `--repo owner/name` to override the repository label for local sessions.
|
|
|
171
175
|
| `ingest` | Load sessions and tool events into DuckDB |
|
|
172
176
|
| `mine` | Show friction clusters, hotspots, recovery loops, and context blasts |
|
|
173
177
|
| `report` | Emit all findings and patterns as evidence-rich JSON for an agent |
|
|
178
|
+
| `doctor` | Check install, Node, credentials, source stores, and database schema |
|
|
174
179
|
| `label` | Optional: classify targets with a decision model (`intent`, `object_type`, confidence) |
|
|
175
180
|
| `patterns` | Mine structural multi-step friction; labels refine when available |
|
|
176
181
|
| `exact` | Show exact recurring sequences with evidence |
|
|
@@ -185,9 +190,10 @@ trace2rules ingest --format swe-rebench # opt into the public dataset
|
|
|
185
190
|
trace2rules ingest --source PATH --limit 200
|
|
186
191
|
trace2rules synthesize
|
|
187
192
|
trace2rules synthesize --offline
|
|
193
|
+
trace2rules synthesize --max-rules 10
|
|
194
|
+
trace2rules synthesize --out-dir ./proposals
|
|
188
195
|
trace2rules synthesize --repo owner/name --out AGENTS.proposed.md
|
|
189
196
|
trace2rules synthesize --repo owner/name --out -
|
|
190
|
-
trace2rules synthesize --out-dir ./proposals
|
|
191
197
|
```
|
|
192
198
|
|
|
193
199
|
`--label` auto-detects an authenticated `wrangler` first, then `GEMINI_API_KEY`/`GOOGLE_API_KEY`, then `OPENAI_API_KEY`. It is opt-in because a full local run can have thousands of unique targets.
|
|
@@ -240,7 +246,7 @@ trace2rules patterns --min-confidence 0.4
|
|
|
240
246
|
|
|
241
247
|
Cloudflare models authenticate through `wrangler auth token --json` and `wrangler whoami --json`; the first account is used. Gemini and OpenAI use their standard keys. Labeling defaults to 270 requests/minute with 6 workers and retries 429/5xx responses.
|
|
242
248
|
|
|
243
|
-
`label` sends the tool name, action type, normalized target, and a raw command example to the selected provider. Do not run
|
|
249
|
+
`label` sends the tool name, action type, redacted normalized target, and a redacted raw command example to the selected provider. The local DuckDB and `report` output remain unredacted. Do not run external labeling or synthesis on private sessions without reviewing the redaction and provider policy.
|
|
244
250
|
|
|
245
251
|
## Configuration
|
|
246
252
|
|
|
@@ -2,8 +2,13 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
|
+
import platform
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
5
9
|
from dataclasses import asdict
|
|
6
10
|
from datetime import datetime, timezone
|
|
11
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
12
|
from pathlib import Path
|
|
8
13
|
from typing import Annotated
|
|
9
14
|
|
|
@@ -12,14 +17,13 @@ import typer
|
|
|
12
17
|
from dotenv import load_dotenv
|
|
13
18
|
from rich.console import Console
|
|
14
19
|
from rich.markup import escape
|
|
20
|
+
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
15
21
|
from rich.table import Table
|
|
16
22
|
|
|
17
|
-
from .cluster import exact_sequences, mine as mine_friction
|
|
18
23
|
from .db import DB_PATH, connect
|
|
19
24
|
from .ingest import DEFAULT_REPOS, SOURCE, ingest as ingest_traces
|
|
20
25
|
from .labels import label_targets
|
|
21
26
|
from .patterns import mine_sequences
|
|
22
|
-
from .synthesize import EXACT_LENGTHS, synthesize as synthesize_rules
|
|
23
27
|
|
|
24
28
|
load_dotenv(Path.cwd() / ".env")
|
|
25
29
|
|
|
@@ -64,59 +68,69 @@ def _ingest_command(
|
|
|
64
68
|
) -> None:
|
|
65
69
|
fmt = format.strip().lower()
|
|
66
70
|
sources = source or []
|
|
71
|
+
failed_sources: list[tuple[str, Exception]] = []
|
|
67
72
|
if repos and fmt != "swe-rebench":
|
|
68
73
|
raise typer.BadParameter("--repos is only valid with --format swe-rebench.")
|
|
69
74
|
if fmt == "swe-rebench":
|
|
70
75
|
prefixes = repos or DEFAULT_REPOS
|
|
71
76
|
selected_sources = sources or [SOURCE]
|
|
72
77
|
stats = []
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
78
|
+
for selected_source in selected_sources:
|
|
79
|
+
if selected_source.startswith(("hf://", "http://", "https://")):
|
|
80
|
+
console.print("[dim]Scanning remote parquet; first run downloads metadata/data...[/dim]")
|
|
81
|
+
try:
|
|
77
82
|
stats.extend(
|
|
78
83
|
ingest_traces(
|
|
79
84
|
prefixes, limit=limit, db_path=db, source=selected_source
|
|
80
85
|
)
|
|
81
86
|
)
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
raise typer.Exit(1) from None
|
|
87
|
+
except (duckdb.Error, OSError, RuntimeError) as error:
|
|
88
|
+
failed_sources.append((selected_source, error))
|
|
85
89
|
elif fmt == "swe-chat":
|
|
86
90
|
from .sources.swe_chat import ingest_swe_chat
|
|
87
91
|
|
|
88
92
|
selected_sources = sources or ["data/swe-chat"]
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
93
|
+
stats = []
|
|
94
|
+
for selected_source in selected_sources:
|
|
95
|
+
try:
|
|
96
|
+
stats.append(
|
|
97
|
+
ingest_swe_chat(
|
|
98
|
+
selected_source,
|
|
99
|
+
db,
|
|
100
|
+
limit=limit or None,
|
|
101
|
+
repo=repo,
|
|
102
|
+
)
|
|
96
103
|
)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
except (FileNotFoundError, ValueError, RuntimeError) as error:
|
|
100
|
-
console.print(f"[red]{escape(str(error))}[/red]")
|
|
101
|
-
raise typer.Exit(1) from None
|
|
104
|
+
except (FileNotFoundError, ValueError, RuntimeError) as error:
|
|
105
|
+
failed_sources.append((selected_source, error))
|
|
102
106
|
elif fmt in {"agent-trajectory", "agent"}:
|
|
103
107
|
from .sources.agent_trajectory import ingest_agent_sessions
|
|
104
108
|
|
|
105
109
|
selected_sources = sources or [str(Path.home())]
|
|
106
110
|
stats = []
|
|
107
|
-
|
|
111
|
+
progress = Progress(
|
|
112
|
+
SpinnerColumn(),
|
|
113
|
+
TextColumn("[progress.description]{task.description}"),
|
|
114
|
+
BarColumn(),
|
|
115
|
+
TextColumn("{task.completed}/{task.total}"),
|
|
116
|
+
TimeElapsedColumn(),
|
|
117
|
+
console=Console(stderr=True),
|
|
118
|
+
transient=False,
|
|
119
|
+
)
|
|
120
|
+
with progress:
|
|
108
121
|
for selected_source in selected_sources:
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
122
|
+
try:
|
|
123
|
+
stats.extend(
|
|
124
|
+
ingest_agent_sessions(
|
|
125
|
+
selected_source,
|
|
126
|
+
db,
|
|
127
|
+
limit=limit or None,
|
|
128
|
+
repo=repo,
|
|
129
|
+
progress=progress,
|
|
130
|
+
)
|
|
115
131
|
)
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
console.print(f"[red]{escape(str(error))}[/red]")
|
|
119
|
-
raise typer.Exit(1) from None
|
|
132
|
+
except (FileNotFoundError, ValueError, RuntimeError) as error:
|
|
133
|
+
failed_sources.append((selected_source, error))
|
|
120
134
|
else:
|
|
121
135
|
raise typer.BadParameter(
|
|
122
136
|
"Unknown format. Use swe-rebench, swe-chat, or agent-trajectory."
|
|
@@ -136,6 +150,12 @@ def _ingest_command(
|
|
|
136
150
|
for stat in stats:
|
|
137
151
|
if stat.matched == 0:
|
|
138
152
|
console.print(f"[yellow]No sessions found for {stat.prefix!r}.[/yellow]")
|
|
153
|
+
for failed_source, error in failed_sources:
|
|
154
|
+
console.print(
|
|
155
|
+
f"[red]Failed {escape(str(failed_source))}: {escape(str(error))}[/red]"
|
|
156
|
+
)
|
|
157
|
+
if failed_sources and not stats:
|
|
158
|
+
raise typer.Exit(1)
|
|
139
159
|
if label:
|
|
140
160
|
from .labels import detect_model
|
|
141
161
|
|
|
@@ -281,6 +301,7 @@ def _mine_command(
|
|
|
281
301
|
console.print("[red]No sessions in the database. Run `trace2rules ingest` first.[/red]")
|
|
282
302
|
raise typer.Exit(1)
|
|
283
303
|
from .detectors import session_summary
|
|
304
|
+
from .cluster import mine as mine_friction
|
|
284
305
|
|
|
285
306
|
_print_summary(session_summary(con))
|
|
286
307
|
mined = mine_friction(con, top=top)
|
|
@@ -318,6 +339,9 @@ def _report_command(
|
|
|
318
339
|
"SELECT repo, count(*) FROM sessions GROUP BY repo ORDER BY repo"
|
|
319
340
|
).fetchall()
|
|
320
341
|
|
|
342
|
+
from .cluster import exact_sequences, mine as mine_friction
|
|
343
|
+
from .synthesize import EXACT_LENGTHS
|
|
344
|
+
|
|
321
345
|
mined = mine_friction(con, top=top)
|
|
322
346
|
findings = [
|
|
323
347
|
finding
|
|
@@ -368,6 +392,127 @@ def _report_command(
|
|
|
368
392
|
app.command("report")(_report_command)
|
|
369
393
|
|
|
370
394
|
|
|
395
|
+
def _doctor_command(
|
|
396
|
+
db: Annotated[Path, typer.Option(help="DuckDB database path.")] = DB_PATH,
|
|
397
|
+
) -> None:
|
|
398
|
+
checks: list[tuple[str, str, str]] = []
|
|
399
|
+
|
|
400
|
+
py_ok = sys.version_info >= (3, 12)
|
|
401
|
+
checks.append(("Python", "ok" if py_ok else "fail", platform.python_version()))
|
|
402
|
+
|
|
403
|
+
try:
|
|
404
|
+
package_version = version("trace2rules")
|
|
405
|
+
checks.append(("trace2rules", "ok", package_version))
|
|
406
|
+
except PackageNotFoundError:
|
|
407
|
+
checks.append(("trace2rules", "warn", "not installed as a package"))
|
|
408
|
+
|
|
409
|
+
node = shutil.which("node")
|
|
410
|
+
if not node:
|
|
411
|
+
checks.append(("Node.js", "warn", "not found; non-OpenCode local sources need it"))
|
|
412
|
+
else:
|
|
413
|
+
try:
|
|
414
|
+
node_version = subprocess.run(
|
|
415
|
+
[node, "--version"],
|
|
416
|
+
capture_output=True,
|
|
417
|
+
text=True,
|
|
418
|
+
timeout=5,
|
|
419
|
+
check=False,
|
|
420
|
+
).stdout.strip()
|
|
421
|
+
except OSError:
|
|
422
|
+
node_version = "unavailable"
|
|
423
|
+
checks.append(("Node.js", "ok", node_version or node))
|
|
424
|
+
|
|
425
|
+
try:
|
|
426
|
+
from .labels import _wrangler_ready, detect_model
|
|
427
|
+
|
|
428
|
+
wrangler_ready = _wrangler_ready()
|
|
429
|
+
checks.append(
|
|
430
|
+
(
|
|
431
|
+
"wrangler",
|
|
432
|
+
"ok" if wrangler_ready else "warn",
|
|
433
|
+
"authenticated" if wrangler_ready else "not authenticated",
|
|
434
|
+
)
|
|
435
|
+
)
|
|
436
|
+
decision_model = detect_model()
|
|
437
|
+
checks.append(
|
|
438
|
+
(
|
|
439
|
+
"decision backend",
|
|
440
|
+
"ok" if decision_model else "warn",
|
|
441
|
+
decision_model or "none detected",
|
|
442
|
+
)
|
|
443
|
+
)
|
|
444
|
+
except Exception as error: # noqa: BLE001 - doctor must not fail on probes
|
|
445
|
+
checks.append(("decision backend", "warn", str(error)))
|
|
446
|
+
|
|
447
|
+
for key in ("TRACE2RULES_LLM_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY"):
|
|
448
|
+
checks.append(
|
|
449
|
+
(key, "ok" if os.getenv(key) else "warn", "set" if os.getenv(key) else "unset")
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
from .sources.agent_trajectory import _source_roots
|
|
453
|
+
|
|
454
|
+
try:
|
|
455
|
+
roots = _source_roots(Path.home())
|
|
456
|
+
detail = ", ".join(f"{name}:{path}" for name, path in roots) or "none found"
|
|
457
|
+
checks.append(
|
|
458
|
+
("local sources", "ok" if roots else "warn", detail)
|
|
459
|
+
)
|
|
460
|
+
except OSError as error:
|
|
461
|
+
checks.append(("local sources", "warn", str(error)))
|
|
462
|
+
|
|
463
|
+
parent = Path(db).parent
|
|
464
|
+
try:
|
|
465
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
466
|
+
writable = os.access(parent, os.W_OK)
|
|
467
|
+
except OSError:
|
|
468
|
+
writable = False
|
|
469
|
+
checks.append(
|
|
470
|
+
("database directory", "ok" if writable else "fail", str(parent))
|
|
471
|
+
)
|
|
472
|
+
if Path(db).exists():
|
|
473
|
+
try:
|
|
474
|
+
con = duckdb.connect(str(db), read_only=True)
|
|
475
|
+
try:
|
|
476
|
+
tables = {
|
|
477
|
+
row[0]
|
|
478
|
+
for row in con.execute(
|
|
479
|
+
"SELECT table_name FROM information_schema.tables"
|
|
480
|
+
).fetchall()
|
|
481
|
+
}
|
|
482
|
+
columns = {
|
|
483
|
+
row[1]
|
|
484
|
+
for row in con.execute("PRAGMA table_info('tool_events')").fetchall()
|
|
485
|
+
}
|
|
486
|
+
if "tool_events" not in tables:
|
|
487
|
+
status, detail = "fail", "missing tool_events"
|
|
488
|
+
elif "segment_idx" not in columns:
|
|
489
|
+
status, detail = "warn", "schema needs re-ingest/migration"
|
|
490
|
+
else:
|
|
491
|
+
status, detail = "ok", "schema ready"
|
|
492
|
+
checks.append(("database", status, detail))
|
|
493
|
+
finally:
|
|
494
|
+
con.close()
|
|
495
|
+
except Exception as error: # noqa: BLE001 - report lock/permission issues
|
|
496
|
+
checks.append(("database", "warn", str(error)))
|
|
497
|
+
else:
|
|
498
|
+
checks.append(("database", "warn", f"not created yet: {db}"))
|
|
499
|
+
|
|
500
|
+
table = Table(title="trace2rules doctor", header_style="bold")
|
|
501
|
+
table.add_column("Check")
|
|
502
|
+
table.add_column("Status")
|
|
503
|
+
table.add_column("Detail", overflow="fold")
|
|
504
|
+
for check, status, detail in checks:
|
|
505
|
+
style = {"ok": "green", "warn": "yellow", "fail": "red"}.get(status, "")
|
|
506
|
+
table.add_row(check, f"[{style}]{status}[/{style}]", escape(detail))
|
|
507
|
+
console.print(table)
|
|
508
|
+
|
|
509
|
+
if not py_ok or not writable:
|
|
510
|
+
raise typer.Exit(1)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
app.command("doctor")(_doctor_command)
|
|
514
|
+
|
|
515
|
+
|
|
371
516
|
def _label_command(
|
|
372
517
|
limit: Annotated[int, typer.Option(help="Max pending targets; 0 = all.")] = 0,
|
|
373
518
|
rpm: Annotated[int, typer.Option(help="Requests per minute cap.")] = 270,
|
|
@@ -456,6 +601,8 @@ def _exact_command(
|
|
|
456
601
|
) -> None:
|
|
457
602
|
con = connect(db)
|
|
458
603
|
try:
|
|
604
|
+
from .cluster import exact_sequences
|
|
605
|
+
|
|
459
606
|
findings = exact_sequences(
|
|
460
607
|
con,
|
|
461
608
|
length=length,
|
|
@@ -500,15 +647,27 @@ app.command("exact")(_exact_command)
|
|
|
500
647
|
|
|
501
648
|
def _synthesize_command(
|
|
502
649
|
out: Annotated[Path | None, typer.Option("--out", "-o", help="Output path for a single --repo; use '-' for stdout.")] = None,
|
|
503
|
-
out_dir: Annotated[
|
|
650
|
+
out_dir: Annotated[
|
|
651
|
+
Path | None,
|
|
652
|
+
typer.Option("--out-dir", help="Directory for per-repo AGENTS files."),
|
|
653
|
+
] = None,
|
|
504
654
|
repo: Annotated[str | None, typer.Option("--repo", help="Synthesize only this repo.")] = None,
|
|
505
655
|
top: Annotated[int, typer.Option(help="Findings to pass to synthesis.")] = 25,
|
|
656
|
+
max_rules: Annotated[
|
|
657
|
+
int,
|
|
658
|
+
typer.Option("--max-rules", min=1, max=20, help="Maximum rules per repo."),
|
|
659
|
+
] = 5,
|
|
506
660
|
offline: Annotated[bool, typer.Option("--offline", help="Skip the LLM and use deterministic rules.")] = False,
|
|
507
661
|
db: Annotated[Path, typer.Option(help="DuckDB database path.")] = DB_PATH,
|
|
508
662
|
) -> None:
|
|
509
663
|
if out is not None and repo is None:
|
|
510
664
|
raise typer.BadParameter("--out requires --repo; use --out-dir for multiple repos.")
|
|
511
|
-
|
|
665
|
+
stdout_mode = out is None and out_dir is None
|
|
666
|
+
cmd_console = (
|
|
667
|
+
Console(stderr=True)
|
|
668
|
+
if stdout_mode or (out is not None and str(out) == "-")
|
|
669
|
+
else console
|
|
670
|
+
)
|
|
512
671
|
con = connect(db)
|
|
513
672
|
try:
|
|
514
673
|
session_count = con.execute("SELECT count(*) FROM sessions").fetchone()[0]
|
|
@@ -516,12 +675,15 @@ def _synthesize_command(
|
|
|
516
675
|
cmd_console.print("[red]No sessions in the database. Run `trace2rules ingest` first.[/red]")
|
|
517
676
|
raise typer.Exit(1)
|
|
518
677
|
try:
|
|
678
|
+
from .synthesize import synthesize as synthesize_rules
|
|
679
|
+
|
|
519
680
|
results = synthesize_rules(
|
|
520
681
|
con,
|
|
521
682
|
out_path=out,
|
|
522
683
|
out_dir=out_dir,
|
|
523
684
|
repo=repo,
|
|
524
685
|
top=top,
|
|
686
|
+
max_rules=max_rules,
|
|
525
687
|
offline=offline,
|
|
526
688
|
console=cmd_console,
|
|
527
689
|
)
|
|
@@ -27,6 +27,7 @@ class Finding:
|
|
|
27
27
|
intent_confidence: float = 0.0
|
|
28
28
|
object_confidence: float = 0.0
|
|
29
29
|
event_keys: tuple[tuple[str, str, str], ...] = field(default_factory=tuple)
|
|
30
|
+
edit_sessions: int = 0
|
|
30
31
|
|
|
31
32
|
@property
|
|
32
33
|
def evidence(self) -> str:
|
|
@@ -102,6 +103,7 @@ def hotspots(con: duckdb.DuckDBPyConnection, limit: int | None = None) -> list[F
|
|
|
102
103
|
sessions=sessions,
|
|
103
104
|
tokens=int(wasted or 0),
|
|
104
105
|
detail=f"read-only in {read_only}/{repo_sessions} sessions ({read_only / repo_sessions:.0%})",
|
|
106
|
+
edit_sessions=sessions - read_only,
|
|
105
107
|
)
|
|
106
108
|
for repo, target, sessions, read_only, repo_sessions, wasted in con.execute(query).fetchall()
|
|
107
109
|
]
|
|
@@ -564,6 +564,7 @@ def write_sessions(
|
|
|
564
564
|
"""Write canonical sessions/events, replacing rows with the same ids."""
|
|
565
565
|
session_list = list(sessions)
|
|
566
566
|
con = connect(db_path)
|
|
567
|
+
con.execute("BEGIN TRANSACTION")
|
|
567
568
|
event_count = 0
|
|
568
569
|
try:
|
|
569
570
|
for session in session_list:
|
|
@@ -656,12 +657,16 @@ def write_sessions(
|
|
|
656
657
|
events,
|
|
657
658
|
)
|
|
658
659
|
event_count += len(events)
|
|
660
|
+
con.execute("COMMIT")
|
|
659
661
|
return IngestStats(
|
|
660
662
|
prefix="canonical",
|
|
661
663
|
matched=len(session_list),
|
|
662
664
|
sessions=len(session_list),
|
|
663
665
|
events=event_count,
|
|
664
666
|
)
|
|
667
|
+
except Exception:
|
|
668
|
+
con.execute("ROLLBACK")
|
|
669
|
+
raise
|
|
665
670
|
finally:
|
|
666
671
|
con.close()
|
|
667
672
|
|
|
@@ -14,6 +14,7 @@ from pathlib import Path
|
|
|
14
14
|
import duckdb
|
|
15
15
|
|
|
16
16
|
from .db import DB_PATH, connect
|
|
17
|
+
from .redact import redact_text
|
|
17
18
|
|
|
18
19
|
DEFAULT_MODEL = "clef-flash"
|
|
19
20
|
DEFAULT_RPM = 270
|
|
@@ -355,8 +356,8 @@ def _state(row: tuple[str, str, str, str]) -> dict:
|
|
|
355
356
|
return {
|
|
356
357
|
"tool": tool_name,
|
|
357
358
|
"action_type": action_type,
|
|
358
|
-
"normalized_target": target[:500],
|
|
359
|
-
"raw_example": (raw_example or "").replace("\x00", " ")[:500],
|
|
359
|
+
"normalized_target": redact_text(target[:500]),
|
|
360
|
+
"raw_example": redact_text((raw_example or "").replace("\x00", " ")[:500]),
|
|
360
361
|
}
|
|
361
362
|
|
|
362
363
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
_PATTERNS = (
|
|
6
|
+
(
|
|
7
|
+
re.compile(
|
|
8
|
+
r"(?i)((?:api[_-]?key|access[_-]?token|auth[_-]?token|secret|"
|
|
9
|
+
r"password|passwd|pwd)\s*[:=]\s*)"
|
|
10
|
+
r"([^\s\"'`]+)"
|
|
11
|
+
),
|
|
12
|
+
r"\1<redacted>",
|
|
13
|
+
),
|
|
14
|
+
(
|
|
15
|
+
re.compile(r"(?i)(authorization:\s*bearer\s+)[A-Za-z0-9._~+/=-]+"),
|
|
16
|
+
r"\1<redacted>",
|
|
17
|
+
),
|
|
18
|
+
(
|
|
19
|
+
re.compile(
|
|
20
|
+
r"(?i)(?<![A-Za-z0-9_-])("
|
|
21
|
+
r"sk-[A-Za-z0-9_-]{16,}|"
|
|
22
|
+
r"gh[pousr]_[A-Za-z0-9]{20,}|"
|
|
23
|
+
r"github_pat_[A-Za-z0-9_]{20,}|"
|
|
24
|
+
r"AKIA[0-9A-Z]{16}|"
|
|
25
|
+
r"AIza[0-9A-Za-z_-]{30,}|"
|
|
26
|
+
r"xox[baprs]-[A-Za-z0-9-]{10,}"
|
|
27
|
+
r")"
|
|
28
|
+
),
|
|
29
|
+
"<redacted>",
|
|
30
|
+
),
|
|
31
|
+
(
|
|
32
|
+
re.compile(
|
|
33
|
+
r"(?i)(https?://[^/\s:@]+):[^@\s]+@"
|
|
34
|
+
),
|
|
35
|
+
r"\1:<redacted>@",
|
|
36
|
+
),
|
|
37
|
+
(
|
|
38
|
+
re.compile(r"(?i)([?&](?:token|key|secret|password|signature)=)[^&\s]+"),
|
|
39
|
+
r"\1<redacted>",
|
|
40
|
+
),
|
|
41
|
+
(
|
|
42
|
+
re.compile(
|
|
43
|
+
r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
|
|
44
|
+
re.DOTALL,
|
|
45
|
+
),
|
|
46
|
+
"<redacted-private-key>",
|
|
47
|
+
),
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def redact_text(value: str) -> str:
|
|
52
|
+
text = value
|
|
53
|
+
for pattern, replacement in _PATTERNS:
|
|
54
|
+
text = pattern.sub(replacement, text)
|
|
55
|
+
return text
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def redact_value(value: object) -> object:
|
|
59
|
+
if isinstance(value, str):
|
|
60
|
+
return redact_text(value)
|
|
61
|
+
if isinstance(value, list):
|
|
62
|
+
return [redact_value(item) for item in value]
|
|
63
|
+
if isinstance(value, tuple):
|
|
64
|
+
return tuple(redact_value(item) for item in value)
|
|
65
|
+
if isinstance(value, dict):
|
|
66
|
+
return {key: redact_value(item) for key, item in value.items()}
|
|
67
|
+
return value
|
|
@@ -2,9 +2,12 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import hashlib
|
|
4
4
|
import json
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
5
6
|
from dataclasses import dataclass, field
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
|
|
9
|
+
from rich.progress import Progress
|
|
10
|
+
|
|
8
11
|
from ..models import Session, ToolCall
|
|
9
12
|
from .common import classify_tool_call, parse_arguments, target_from_args
|
|
10
13
|
|
|
@@ -80,6 +83,83 @@ def _walk_tool_ids(value: object):
|
|
|
80
83
|
yield from _walk_tool_ids(child)
|
|
81
84
|
|
|
82
85
|
|
|
86
|
+
def _tool_ids(row: dict, source: str) -> tuple[str, ...]:
|
|
87
|
+
if source in {"omp", "pi"}:
|
|
88
|
+
message = row.get("message") or {}
|
|
89
|
+
if message.get("role") != "assistant":
|
|
90
|
+
return ()
|
|
91
|
+
content = message.get("content")
|
|
92
|
+
if not isinstance(content, list):
|
|
93
|
+
return ()
|
|
94
|
+
return tuple(
|
|
95
|
+
str(part["id"])
|
|
96
|
+
for part in content
|
|
97
|
+
if isinstance(part, dict)
|
|
98
|
+
and part.get("type") in {"toolCall", "tool_call"}
|
|
99
|
+
and part.get("id")
|
|
100
|
+
)
|
|
101
|
+
if source == "codex":
|
|
102
|
+
payload = row.get("payload") or {}
|
|
103
|
+
if payload.get("type") not in TOOL_CALL_TYPES:
|
|
104
|
+
return ()
|
|
105
|
+
call_id = payload.get("id") or payload.get("call_id")
|
|
106
|
+
return (str(call_id),) if call_id else ()
|
|
107
|
+
if source == "claude-code":
|
|
108
|
+
message = row.get("message") or {}
|
|
109
|
+
content = message.get("content")
|
|
110
|
+
if not isinstance(content, list):
|
|
111
|
+
return ()
|
|
112
|
+
return tuple(
|
|
113
|
+
str(part["id"])
|
|
114
|
+
for part in content
|
|
115
|
+
if isinstance(part, dict)
|
|
116
|
+
and part.get("type") in {"tool_use", "toolCall", "tool_call"}
|
|
117
|
+
and part.get("id")
|
|
118
|
+
)
|
|
119
|
+
return tuple(_walk_tool_ids(row))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _relevant_line(line: str, source: str) -> bool:
|
|
123
|
+
if source in {"omp", "pi"}:
|
|
124
|
+
return any(
|
|
125
|
+
marker in line
|
|
126
|
+
for marker in (
|
|
127
|
+
'"type":"session"',
|
|
128
|
+
'"type": "session"',
|
|
129
|
+
'"type":"compaction"',
|
|
130
|
+
'"type": "compaction"',
|
|
131
|
+
'"toolCall"',
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
if source == "codex":
|
|
135
|
+
return any(
|
|
136
|
+
marker in line
|
|
137
|
+
for marker in (
|
|
138
|
+
'"type":"session_meta"',
|
|
139
|
+
'"type": "session_meta"',
|
|
140
|
+
'"type":"compaction"',
|
|
141
|
+
'"type": "compaction"',
|
|
142
|
+
'"function_call"',
|
|
143
|
+
'"custom_tool_call"',
|
|
144
|
+
'"local_shell_call"',
|
|
145
|
+
'"web_search_call"',
|
|
146
|
+
'"mcp_tool_call"',
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
if source == "claude-code":
|
|
150
|
+
return any(
|
|
151
|
+
marker in line
|
|
152
|
+
for marker in (
|
|
153
|
+
'"sessionId"',
|
|
154
|
+
'"tool_use"',
|
|
155
|
+
'"type":"summary"',
|
|
156
|
+
'"type": "summary"',
|
|
157
|
+
'"compaction"',
|
|
158
|
+
)
|
|
159
|
+
)
|
|
160
|
+
return True
|
|
161
|
+
|
|
162
|
+
|
|
83
163
|
def extract_transcript_meta(path: Path, source: str, text: str) -> TranscriptMeta:
|
|
84
164
|
"""Extract header id, parent id, and compaction episode boundaries.
|
|
85
165
|
|
|
@@ -91,6 +171,8 @@ def extract_transcript_meta(path: Path, source: str, text: str) -> TranscriptMet
|
|
|
91
171
|
for line in text.splitlines():
|
|
92
172
|
if not line.strip():
|
|
93
173
|
continue
|
|
174
|
+
if not _relevant_line(line, source):
|
|
175
|
+
continue
|
|
94
176
|
try:
|
|
95
177
|
row = json.loads(line)
|
|
96
178
|
except json.JSONDecodeError:
|
|
@@ -126,7 +208,7 @@ def extract_transcript_meta(path: Path, source: str, text: str) -> TranscriptMet
|
|
|
126
208
|
if _is_compaction(row, source):
|
|
127
209
|
segment += 1
|
|
128
210
|
continue
|
|
129
|
-
for call_id in
|
|
211
|
+
for call_id in _tool_ids(row, source):
|
|
130
212
|
meta.segment_of_call[call_id] = segment
|
|
131
213
|
|
|
132
214
|
meta.segments = segment + 1
|
|
@@ -316,6 +398,7 @@ def ingest_agent_sessions(
|
|
|
316
398
|
*,
|
|
317
399
|
limit: int | None = None,
|
|
318
400
|
repo: str | None = None,
|
|
401
|
+
progress: Progress | None = None,
|
|
319
402
|
):
|
|
320
403
|
"""Normalize local agent sessions with @letta-ai/trajectory."""
|
|
321
404
|
try:
|
|
@@ -378,14 +461,37 @@ def ingest_agent_sessions(
|
|
|
378
461
|
"export sessions with `opencode export <session-id>` first."
|
|
379
462
|
)
|
|
380
463
|
continue
|
|
381
|
-
transcripts: list[tuple[Path, str]] = [
|
|
382
|
-
|
|
383
|
-
|
|
464
|
+
transcripts: list[tuple[Path, str]] = []
|
|
465
|
+
read_task = (
|
|
466
|
+
progress.add_task(
|
|
467
|
+
f"Reading {source_name}", total=len(files)
|
|
468
|
+
)
|
|
469
|
+
if progress
|
|
470
|
+
else None
|
|
471
|
+
)
|
|
472
|
+
for path in files:
|
|
473
|
+
try:
|
|
474
|
+
transcripts.append(
|
|
475
|
+
(path, path.read_text(encoding="utf-8", errors="replace"))
|
|
476
|
+
)
|
|
477
|
+
except OSError as error:
|
|
478
|
+
print(f"Skipping {path}: {error}")
|
|
479
|
+
if progress and read_task is not None:
|
|
480
|
+
progress.update(read_task, advance=1)
|
|
384
481
|
metas_by_id: dict[str, TranscriptMeta] = {}
|
|
385
482
|
transcript_metas: list[tuple[Path, str, TranscriptMeta]] = []
|
|
483
|
+
scan_task = (
|
|
484
|
+
progress.add_task(
|
|
485
|
+
f"Scanning {source_name}", total=len(transcripts)
|
|
486
|
+
)
|
|
487
|
+
if progress
|
|
488
|
+
else None
|
|
489
|
+
)
|
|
386
490
|
for path, text in transcripts:
|
|
387
491
|
meta = extract_transcript_meta(path, source_name, text)
|
|
388
492
|
meta.session_id = meta.session_id or _session_id(path)
|
|
493
|
+
if progress and scan_task is not None:
|
|
494
|
+
progress.update(scan_task, advance=1)
|
|
389
495
|
if meta.session_id in metas_by_id:
|
|
390
496
|
continue
|
|
391
497
|
metas_by_id[meta.session_id] = meta
|
|
@@ -393,12 +499,17 @@ def ingest_agent_sessions(
|
|
|
393
499
|
for meta in metas_by_id.values():
|
|
394
500
|
meta.instance_id = _resolve_instance(meta.session_id, metas_by_id)
|
|
395
501
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
502
|
+
task_id = (
|
|
503
|
+
progress.add_task(
|
|
504
|
+
f"Ingesting {source_name}", total=len(transcript_metas)
|
|
505
|
+
)
|
|
506
|
+
if progress
|
|
507
|
+
else None
|
|
508
|
+
)
|
|
509
|
+
def normalize_batch(batch):
|
|
510
|
+
normalized = []
|
|
400
511
|
try:
|
|
401
|
-
|
|
512
|
+
return list(
|
|
402
513
|
normalize_many(
|
|
403
514
|
[
|
|
404
515
|
{
|
|
@@ -413,7 +524,7 @@ def ingest_agent_sessions(
|
|
|
413
524
|
except NormalizationError:
|
|
414
525
|
for path, text, _ in batch:
|
|
415
526
|
try:
|
|
416
|
-
|
|
527
|
+
normalized.extend(
|
|
417
528
|
[
|
|
418
529
|
normalize_many(
|
|
419
530
|
[
|
|
@@ -428,7 +539,30 @@ def ingest_agent_sessions(
|
|
|
428
539
|
)
|
|
429
540
|
except NormalizationError as error:
|
|
430
541
|
print(f"Skipping {path}: {error}")
|
|
431
|
-
|
|
542
|
+
normalized.append({"records": [], "diagnostics": []})
|
|
543
|
+
return normalized
|
|
544
|
+
|
|
545
|
+
BATCH = 16
|
|
546
|
+
batches = [
|
|
547
|
+
transcript_metas[start : start + BATCH]
|
|
548
|
+
for start in range(0, len(transcript_metas), BATCH)
|
|
549
|
+
]
|
|
550
|
+
ordered_results: list[list[dict] | None] = [None] * len(batches)
|
|
551
|
+
workers = min(3, len(batches)) if batches else 1
|
|
552
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
553
|
+
futures = {
|
|
554
|
+
pool.submit(normalize_batch, batch): index
|
|
555
|
+
for index, batch in enumerate(batches)
|
|
556
|
+
}
|
|
557
|
+
for future in as_completed(futures):
|
|
558
|
+
index = futures[future]
|
|
559
|
+
ordered_results[index] = future.result()
|
|
560
|
+
if progress and task_id is not None:
|
|
561
|
+
progress.update(task_id, advance=len(batches[index]))
|
|
562
|
+
results: list[dict] = []
|
|
563
|
+
for batch_results in ordered_results:
|
|
564
|
+
if batch_results:
|
|
565
|
+
results.extend(batch_results)
|
|
432
566
|
for (path, _text, meta), result in zip(transcript_metas, results):
|
|
433
567
|
if not result.get("records"):
|
|
434
568
|
continue
|
|
@@ -452,5 +586,12 @@ def ingest_agent_sessions(
|
|
|
452
586
|
"agent-trajectory requires Node.js 20 or newer on PATH."
|
|
453
587
|
) from error
|
|
454
588
|
|
|
589
|
+
write_task = (
|
|
590
|
+
progress.add_task("Writing sessions", total=len(all_sessions))
|
|
591
|
+
if progress
|
|
592
|
+
else None
|
|
593
|
+
)
|
|
455
594
|
write_sessions(db_path, all_sessions)
|
|
595
|
+
if progress and write_task is not None:
|
|
596
|
+
progress.update(write_task, completed=len(all_sessions))
|
|
456
597
|
return stats
|
|
@@ -15,11 +15,11 @@ from rich.markup import escape
|
|
|
15
15
|
from rich.table import Table
|
|
16
16
|
|
|
17
17
|
from .detectors import Finding
|
|
18
|
+
from .redact import redact_value
|
|
18
19
|
|
|
19
20
|
MAX_RULES = 5
|
|
20
21
|
MAX_LINES = 60
|
|
21
22
|
MAX_CHARS = 2500
|
|
22
|
-
OUT_PATH = Path("AGENTS.proposed.md")
|
|
23
23
|
EXACT_LENGTHS = (3, 4, 5)
|
|
24
24
|
|
|
25
25
|
GENERATED_VERIFIER_RE = re.compile(
|
|
@@ -260,8 +260,14 @@ def _make_client(key: str, base: str | None, model: str):
|
|
|
260
260
|
return client, model
|
|
261
261
|
|
|
262
262
|
|
|
263
|
-
def _prompt(
|
|
264
|
-
|
|
263
|
+
def _prompt(
|
|
264
|
+
findings: list[Finding],
|
|
265
|
+
repo: str,
|
|
266
|
+
labels: list[dict],
|
|
267
|
+
max_rules: int = MAX_RULES,
|
|
268
|
+
) -> str:
|
|
269
|
+
payload = redact_value(
|
|
270
|
+
[
|
|
265
271
|
{
|
|
266
272
|
"kind": finding.kind,
|
|
267
273
|
"sessions": finding.sessions,
|
|
@@ -277,17 +283,18 @@ def _prompt(findings: list[Finding], repo: str, labels: list[dict]) -> str:
|
|
|
277
283
|
"object_type": finding.object_type,
|
|
278
284
|
}
|
|
279
285
|
for finding in findings
|
|
280
|
-
|
|
286
|
+
]
|
|
287
|
+
)
|
|
281
288
|
return f"""Repo: {repo}
|
|
282
289
|
|
|
283
290
|
Findings from mined agent trajectories for this repo only (JSON):
|
|
284
291
|
{json.dumps(payload, indent=2)}
|
|
285
292
|
|
|
286
293
|
Semantic target labels observed for this repo (clef/decision-model annotations, JSON):
|
|
287
|
-
{json.dumps(labels, indent=2)}
|
|
294
|
+
{json.dumps(redact_value(labels), indent=2)}
|
|
288
295
|
|
|
289
296
|
Write AGENTS.md rules for this repository. Requirements:
|
|
290
|
-
- Emit 1-
|
|
297
|
+
- Emit 1-{max_rules} declarative rules, not task-specific issue descriptions.
|
|
291
298
|
- Categories: test_command, architecture_map, context_hygiene, convention.
|
|
292
299
|
- rule_markdown: 1-2 imperative sentences, concrete command or file names.
|
|
293
300
|
- evidence_summary: sessions affected and approximate setup/rework token cost.
|
|
@@ -295,14 +302,14 @@ Write AGENTS.md rules for this repository. Requirements:
|
|
|
295
302
|
- Ignore task-specific verifier scripts, one-off filenames, and normal edit -> test-fail loops.
|
|
296
303
|
- For `recovery` findings, turn the failed -> successful transition into a convention rule.
|
|
297
304
|
- If any finding has `paired_instances >= 2` and `delta_rework >= 1`, include one `convention` rule: read the exception signature; if the same signature repeats after an edit, fix the product code or test scope instead of editing the harness.
|
|
298
|
-
- Total output must stay under
|
|
305
|
+
- Total output must stay under {max_rules} rules, {max(MAX_LINES, max_rules * 10)} lines, and {max(MAX_CHARS, max_rules * 500):,} characters.
|
|
299
306
|
|
|
300
307
|
Return JSON only, shaped exactly as:
|
|
301
308
|
{{"repo": "{repo}", "rules": [{{"category": "...", "rule_markdown": "...", "evidence_summary": "...", "estimated_tokens_saved_per_session": 0}}]}}"""
|
|
302
309
|
|
|
303
310
|
|
|
304
311
|
def _llm_synthesis(
|
|
305
|
-
findings: list[Finding], repo: str, labels: list[dict]
|
|
312
|
+
findings: list[Finding], repo: str, labels: list[dict], max_rules: int = MAX_RULES
|
|
306
313
|
) -> Synthesis | None:
|
|
307
314
|
resolved = _client()
|
|
308
315
|
if resolved is None:
|
|
@@ -317,7 +324,7 @@ def _llm_synthesis(
|
|
|
317
324
|
"role": "system",
|
|
318
325
|
"content": "You convert observed coding-agent friction into terse, repo-level AGENTS.md rules.",
|
|
319
326
|
},
|
|
320
|
-
{"role": "user", "content": _prompt(findings, repo, labels)},
|
|
327
|
+
{"role": "user", "content": _prompt(findings, repo, labels, max_rules)},
|
|
321
328
|
],
|
|
322
329
|
)
|
|
323
330
|
content = (response.choices[0].message.content or "").strip()
|
|
@@ -329,15 +336,41 @@ def _is_path_target(title: str) -> bool:
|
|
|
329
336
|
return bool(title) and " " not in title and ("/" in title or "." in title)
|
|
330
337
|
|
|
331
338
|
|
|
332
|
-
|
|
339
|
+
DOC_TARGET_RE = re.compile(
|
|
340
|
+
r"(?i)(^|/)(readme|docs?|changelog|contributing|license)(\.|/|$)"
|
|
341
|
+
)
|
|
342
|
+
INSTALL_RECOVERY_RE = re.compile(
|
|
343
|
+
r"(?i)\b(pip|uv|poetry)\b.*\b(install|sync)\b|pip install .* -e"
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _is_doc_target(title: str) -> bool:
|
|
348
|
+
return bool(DOC_TARGET_RE.search(title))
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _offline_synthesis(
|
|
352
|
+
findings: list[Finding], repo: str, max_rules: int = MAX_RULES
|
|
353
|
+
) -> Synthesis:
|
|
333
354
|
rules: list[Rule] = []
|
|
334
355
|
seen: set[tuple[str, str]] = set()
|
|
356
|
+
counts: dict[str, int] = {}
|
|
357
|
+
limits = {
|
|
358
|
+
"test_command": 1,
|
|
359
|
+
"convention": 2,
|
|
360
|
+
"architecture_map": 2,
|
|
361
|
+
"context_hygiene": 1,
|
|
362
|
+
}
|
|
335
363
|
|
|
336
|
-
def add(category: str, markdown: str, evidence: str, tokens: int) -> None:
|
|
337
|
-
key = (category,
|
|
338
|
-
if
|
|
364
|
+
def add(category: str, subject: str, markdown: str, evidence: str, tokens: int) -> None:
|
|
365
|
+
key = (category, subject)
|
|
366
|
+
if (
|
|
367
|
+
key in seen
|
|
368
|
+
or len(rules) >= max_rules
|
|
369
|
+
or counts.get(category, 0) >= limits.get(category, max_rules)
|
|
370
|
+
):
|
|
339
371
|
return
|
|
340
372
|
seen.add(key)
|
|
373
|
+
counts[category] = counts.get(category, 0) + 1
|
|
341
374
|
rules.append(
|
|
342
375
|
Rule(
|
|
343
376
|
category=category,
|
|
@@ -347,18 +380,36 @@ def _offline_synthesis(findings: list[Finding], repo: str) -> Synthesis:
|
|
|
347
380
|
)
|
|
348
381
|
)
|
|
349
382
|
|
|
350
|
-
|
|
383
|
+
def priority(finding: Finding) -> int:
|
|
384
|
+
if finding.kind == "exact" and "pip:editable" in finding.sequence:
|
|
385
|
+
return 0
|
|
386
|
+
if finding.kind == "recovery":
|
|
387
|
+
return 1
|
|
388
|
+
if finding.kind == "cluster":
|
|
389
|
+
return 2
|
|
390
|
+
if finding.kind == "hotspot":
|
|
391
|
+
return 3
|
|
392
|
+
if finding.kind == "context":
|
|
393
|
+
return 4
|
|
394
|
+
return 5
|
|
395
|
+
|
|
396
|
+
for finding in sorted(findings, key=priority):
|
|
397
|
+
if finding.sessions < 3:
|
|
398
|
+
continue
|
|
351
399
|
if finding.intent and finding.intent_confidence < 0.4:
|
|
352
400
|
continue
|
|
401
|
+
|
|
353
402
|
if finding.kind == "exact" and "pip:editable" in finding.sequence:
|
|
354
403
|
if finding.sessions < 10:
|
|
355
404
|
continue
|
|
356
405
|
add(
|
|
357
406
|
"test_command",
|
|
407
|
+
"editable-install",
|
|
358
408
|
"Run `pip install -e .` before import, CLI, or pytest probes; treat pre-install failures as setup failures, not code failures.",
|
|
359
|
-
f"{finding.sessions} sessions; {finding.setup_tax:,} setup tokens.",
|
|
409
|
+
f"{finding.sessions} sessions; {finding.setup_tax:,} setup tokens; {finding.tokens:,} window tokens.",
|
|
360
410
|
finding.setup_tax // max(finding.sessions, 1),
|
|
361
411
|
)
|
|
412
|
+
|
|
362
413
|
elif (
|
|
363
414
|
finding.kind == "exact"
|
|
364
415
|
and finding.paired_instances >= 2
|
|
@@ -366,21 +417,47 @@ def _offline_synthesis(findings: list[Finding], repo: str) -> Synthesis:
|
|
|
366
417
|
):
|
|
367
418
|
add(
|
|
368
419
|
"convention",
|
|
420
|
+
"exception-signature",
|
|
369
421
|
"After a command fails, read the exception signature. If the same signature repeats after an edit, change the product code or test scope instead of editing the harness.",
|
|
370
|
-
f"Paired delta
|
|
422
|
+
f"Paired delta {finding.delta_rework:+.1f} rework in {finding.paired_instances} instances; {finding.cost:,} cost.",
|
|
371
423
|
int(finding.delta_rework * 1000),
|
|
372
424
|
)
|
|
373
|
-
|
|
425
|
+
|
|
426
|
+
elif finding.kind == "recovery":
|
|
427
|
+
recovery = finding.title.split("->", 1)[-1].strip()
|
|
428
|
+
if INSTALL_RECOVERY_RE.search(recovery) and finding.sessions >= 10:
|
|
429
|
+
add(
|
|
430
|
+
"test_command",
|
|
431
|
+
"editable-install",
|
|
432
|
+
"Run `pip install -e .` before import, CLI, or pytest probes; treat pre-install failures as setup failures, not code failures.",
|
|
433
|
+
f"{finding.detail}; {finding.tokens:,} tokens.",
|
|
434
|
+
finding.tokens // max(finding.sessions, 1),
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
elif finding.kind == "cluster":
|
|
438
|
+
example = finding.variants[0] if finding.variants else ""
|
|
439
|
+
if (
|
|
440
|
+
not example
|
|
441
|
+
or ":err" not in finding.sequence
|
|
442
|
+
or not re.search(
|
|
443
|
+
r"(?i)(reproduce|repro|scratch|debug|verify)[\w.-]*\.py",
|
|
444
|
+
example,
|
|
445
|
+
)
|
|
446
|
+
):
|
|
447
|
+
continue
|
|
374
448
|
add(
|
|
375
449
|
"convention",
|
|
376
|
-
f"
|
|
377
|
-
f"{
|
|
378
|
-
finding.
|
|
450
|
+
f"sequence:{_truncate(example, 60)}",
|
|
451
|
+
f"A scripted `{_truncate(example, 120)}` loop appears across {finding.sessions} sessions. Inspect the error before editing or re-running the script.",
|
|
452
|
+
f"{finding.sessions} sessions; {finding.tokens:,} tokens.",
|
|
453
|
+
finding.tokens // max(finding.sessions, 1),
|
|
379
454
|
)
|
|
455
|
+
|
|
380
456
|
elif finding.kind == "hotspot":
|
|
381
|
-
if finding.
|
|
457
|
+
if not _is_path_target(finding.title) or _is_doc_target(finding.title):
|
|
382
458
|
continue
|
|
383
|
-
if
|
|
459
|
+
if finding.edit_sessions > 0:
|
|
460
|
+
# The path is edited somewhere in the repo; do not call it read-only.
|
|
384
461
|
continue
|
|
385
462
|
basename = finding.title.rstrip("/").rsplit("/", 1)[-1]
|
|
386
463
|
if "." in basename:
|
|
@@ -389,29 +466,41 @@ def _offline_synthesis(findings: list[Finding], repo: str) -> Synthesis:
|
|
|
389
466
|
markdown = f"Map `{_truncate(finding.title, 70)}` before editing; it is a read-only reference path."
|
|
390
467
|
add(
|
|
391
468
|
"architecture_map",
|
|
469
|
+
finding.title,
|
|
392
470
|
markdown,
|
|
393
471
|
f"Read-only in {finding.sessions} sessions; {finding.tokens:,} tokens spent.",
|
|
394
472
|
finding.tokens // max(finding.sessions, 1),
|
|
395
473
|
)
|
|
474
|
+
|
|
396
475
|
elif finding.kind == "context":
|
|
397
476
|
if finding.intent == "other":
|
|
398
477
|
continue
|
|
399
|
-
if
|
|
478
|
+
if _is_doc_target(finding.title):
|
|
479
|
+
continue
|
|
480
|
+
if re.search(r"(?i)\b(pip|uv|poetry)\b.*\binstall\b", finding.title):
|
|
400
481
|
continue
|
|
401
482
|
avg = re.search(r"avg (\d+) tokens", finding.detail)
|
|
402
483
|
size = f"{int(avg.group(1)):,}" if avg else "large"
|
|
484
|
+
if " " in finding.title:
|
|
485
|
+
markdown = (
|
|
486
|
+
f"Bound `{_truncate(finding.title, 70)}` output with `-q`, "
|
|
487
|
+
"`| head`/`| tail`, or a narrower selector; it averages "
|
|
488
|
+
f"{size} tokens."
|
|
489
|
+
)
|
|
490
|
+
else:
|
|
491
|
+
markdown = (
|
|
492
|
+
f"Inspect `{_truncate(finding.title, 70)}` by targeted line "
|
|
493
|
+
f"range instead of full-file reads; it averages {size} tokens."
|
|
494
|
+
)
|
|
403
495
|
add(
|
|
404
496
|
"context_hygiene",
|
|
405
|
-
|
|
497
|
+
finding.title,
|
|
498
|
+
markdown,
|
|
406
499
|
f"Average {size} output tokens in {finding.sessions} sessions.",
|
|
407
500
|
finding.tokens // max(finding.sessions, 1),
|
|
408
501
|
)
|
|
409
|
-
elif finding.kind == "recovery":
|
|
410
|
-
# Recovery titles are task-specific command fragments; leave them to
|
|
411
|
-
# the LLM, which can generalize them into a convention rule.
|
|
412
|
-
continue
|
|
413
502
|
|
|
414
|
-
return Synthesis(repo=repo, rules=rules[:
|
|
503
|
+
return Synthesis(repo=repo, rules=rules[:max_rules])
|
|
415
504
|
|
|
416
505
|
|
|
417
506
|
def render(synthesis: Synthesis, repo: str, sessions: int) -> str:
|
|
@@ -429,14 +518,18 @@ def render(synthesis: Synthesis, repo: str, sessions: int) -> str:
|
|
|
429
518
|
return "\n".join(lines).rstrip() + "\n"
|
|
430
519
|
|
|
431
520
|
|
|
432
|
-
def enforce_caps(
|
|
433
|
-
synthesis
|
|
521
|
+
def enforce_caps(
|
|
522
|
+
synthesis: Synthesis, repo: str, sessions: int, max_rules: int = MAX_RULES
|
|
523
|
+
) -> tuple[Synthesis, str]:
|
|
524
|
+
max_lines = max(MAX_LINES, max_rules * 10)
|
|
525
|
+
max_chars = max(MAX_CHARS, max_rules * 500)
|
|
526
|
+
synthesis = Synthesis(repo=synthesis.repo, rules=synthesis.rules[:max_rules])
|
|
434
527
|
while True:
|
|
435
528
|
text = render(synthesis, repo, sessions)
|
|
436
|
-
if len(text) <=
|
|
529
|
+
if len(text) <= max_chars and len(text.splitlines()) <= max_lines:
|
|
437
530
|
return synthesis, text
|
|
438
531
|
if not synthesis.rules:
|
|
439
|
-
return synthesis, text[:
|
|
532
|
+
return synthesis, text[:max_chars]
|
|
440
533
|
synthesis = Synthesis(repo=synthesis.repo, rules=synthesis.rules[:-1])
|
|
441
534
|
|
|
442
535
|
|
|
@@ -447,10 +540,11 @@ def _slug(repo: str) -> str:
|
|
|
447
540
|
def synthesize(
|
|
448
541
|
con,
|
|
449
542
|
out_path: str | Path | None = None,
|
|
450
|
-
out_dir: str | Path =
|
|
543
|
+
out_dir: str | Path | None = None,
|
|
451
544
|
repo: str | None = None,
|
|
452
545
|
top: int = 25,
|
|
453
546
|
offline: bool = False,
|
|
547
|
+
max_rules: int = MAX_RULES,
|
|
454
548
|
console: Console | None = None,
|
|
455
549
|
) -> list[tuple[Path, Synthesis]]:
|
|
456
550
|
from .cluster import exact_sequences, mine
|
|
@@ -463,20 +557,22 @@ def synthesize(
|
|
|
463
557
|
raise ValueError(f"repo not found in database: {repo}")
|
|
464
558
|
|
|
465
559
|
exact: list[Finding] = []
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
560
|
+
with console.status("[cyan]Mining exact sequences and friction clusters...[/cyan]"):
|
|
561
|
+
for length in EXACT_LENGTHS:
|
|
562
|
+
exact.extend(
|
|
563
|
+
exact_sequences(
|
|
564
|
+
con,
|
|
565
|
+
length=length,
|
|
566
|
+
min_sessions=3,
|
|
567
|
+
top=500,
|
|
568
|
+
min_instances=2,
|
|
569
|
+
min_lift=1.5,
|
|
570
|
+
)
|
|
475
571
|
)
|
|
476
|
-
)
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
572
|
+
mined = mine(con, top=None)
|
|
573
|
+
out_dir_path = Path(out_dir) if out_dir else None
|
|
574
|
+
if out_dir_path is not None:
|
|
575
|
+
out_dir_path.mkdir(parents=True, exist_ok=True)
|
|
480
576
|
results: list[tuple[Path, Synthesis]] = []
|
|
481
577
|
|
|
482
578
|
for row in repo_rows:
|
|
@@ -496,12 +592,17 @@ def synthesize(
|
|
|
496
592
|
llm_failed = False
|
|
497
593
|
if not offline:
|
|
498
594
|
try:
|
|
499
|
-
|
|
595
|
+
with console.status(
|
|
596
|
+
f"[cyan]Synthesizing {escape(repo_name)}...[/cyan]"
|
|
597
|
+
):
|
|
598
|
+
synthesis = _llm_synthesis(
|
|
599
|
+
findings, repo_name, labels, max_rules=max_rules
|
|
600
|
+
)
|
|
500
601
|
except Exception as error: # noqa: BLE001 - degrade to deterministic output
|
|
501
602
|
llm_failed = True
|
|
502
603
|
console.print(f"[yellow]LLM synthesis unavailable: {escape(str(error))}[/yellow]")
|
|
503
604
|
if synthesis is None:
|
|
504
|
-
synthesis = _offline_synthesis(findings, repo_name)
|
|
605
|
+
synthesis = _offline_synthesis(findings, repo_name, max_rules=max_rules)
|
|
505
606
|
if offline:
|
|
506
607
|
reason = "--offline"
|
|
507
608
|
elif llm_failed:
|
|
@@ -510,7 +611,9 @@ def synthesize(
|
|
|
510
611
|
reason = "no LLM credentials"
|
|
511
612
|
console.print(f"[yellow]Deterministic rules for {escape(repo_name)} ({reason}).[/yellow]")
|
|
512
613
|
|
|
513
|
-
synthesis, text = enforce_caps(
|
|
614
|
+
synthesis, text = enforce_caps(
|
|
615
|
+
synthesis, repo_name, row["sessions"], max_rules=max_rules
|
|
616
|
+
)
|
|
514
617
|
if not synthesis.rules:
|
|
515
618
|
if repo and out_path is not None and str(out_path) != "-":
|
|
516
619
|
Path(out_path).unlink(missing_ok=True)
|
|
@@ -524,8 +627,14 @@ def synthesize(
|
|
|
524
627
|
results.append((Path("-"), synthesis))
|
|
525
628
|
continue
|
|
526
629
|
path = Path(out_path)
|
|
630
|
+
elif out_dir_path is not None:
|
|
631
|
+
path = out_dir_path / f"AGENTS.{_slug(repo_name)}.proposed.md"
|
|
527
632
|
else:
|
|
528
|
-
|
|
633
|
+
sys.stdout.write(text)
|
|
634
|
+
if not text.endswith("\n"):
|
|
635
|
+
sys.stdout.write("\n")
|
|
636
|
+
results.append((Path("-"), synthesis))
|
|
637
|
+
continue
|
|
529
638
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
530
639
|
path.write_text(text)
|
|
531
640
|
results.append((path, synthesis))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|