agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
"""Import command evidence without retaining desktop content."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections import Counter, defaultdict
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..config import AutError
|
|
12
|
+
from ..observations import ingest, disputed_tool_ids
|
|
13
|
+
from .store import Store, digest
|
|
14
|
+
|
|
15
|
+
MAX_BYTES = 32 * 1024 * 1024
|
|
16
|
+
MAX_EVENTS = 100_000
|
|
17
|
+
DELIVERY = {
|
|
18
|
+
"unknown": "unknown",
|
|
19
|
+
"not_delivered": "safe",
|
|
20
|
+
"delivery_uncertain": "unsafe",
|
|
21
|
+
"delivered_unverified": "unsafe",
|
|
22
|
+
"delivered_verified": "unsafe",
|
|
23
|
+
}
|
|
24
|
+
CODES = set(
|
|
25
|
+
"PERM_DENIED ELEMENT_NOT_FOUND APP_NOT_FOUND ACTION_FAILED ACTION_NOT_SUPPORTED "
|
|
26
|
+
"STALE_REF AMBIGUOUS_TARGET WINDOW_NOT_FOUND PLATFORM_NOT_SUPPORTED TIMEOUT "
|
|
27
|
+
"INVALID_ARGS NOTIFICATION_NOT_FOUND SNAPSHOT_NOT_FOUND SANDBOX_NOT_FOUND "
|
|
28
|
+
"POLICY_DENIED APP_UNRESPONSIVE INTERNAL".split()
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _integer(value: Any) -> bool:
|
|
33
|
+
return type(value) is int and 0 <= value <= 2**63 - 1
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _identity(row: dict[str, Any], seq: str = "seq") -> tuple[int, int, int]:
|
|
37
|
+
values = tuple(row.get(key) for key in ("writer_pid", "writer_proc_start_ms", seq))
|
|
38
|
+
if not all(_integer(value) for value in values):
|
|
39
|
+
raise AutError("desktop: invalid writer or sequence identity")
|
|
40
|
+
return values # type: ignore[return-value]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _read(
|
|
44
|
+
path: Path | None, coverage: Counter[str], budget: list[int], session_id: str | None = None
|
|
45
|
+
) -> list[dict[str, Any]]:
|
|
46
|
+
if path is None:
|
|
47
|
+
return []
|
|
48
|
+
if not path.exists():
|
|
49
|
+
coverage["missing_sources"] += 1
|
|
50
|
+
return []
|
|
51
|
+
if path.is_symlink():
|
|
52
|
+
raise AutError("desktop: symbolic sources are unsupported")
|
|
53
|
+
if path.is_dir():
|
|
54
|
+
files: list[Path] = []
|
|
55
|
+
for entry in path.iterdir():
|
|
56
|
+
if len(files) >= 4096:
|
|
57
|
+
raise AutError("desktop: directory entry cap exceeded")
|
|
58
|
+
files.append(entry)
|
|
59
|
+
if any(not entry.is_file() or entry.is_symlink() for entry in files):
|
|
60
|
+
raise AutError("desktop: expected a directory of raw segment files")
|
|
61
|
+
else:
|
|
62
|
+
files = [path]
|
|
63
|
+
rows: list[dict[str, Any]] = []
|
|
64
|
+
for file in sorted(files):
|
|
65
|
+
with file.open("rb") as stream:
|
|
66
|
+
raw = stream.read(MAX_BYTES - budget[0] + 1)
|
|
67
|
+
budget[0] += len(raw)
|
|
68
|
+
if budget[0] > MAX_BYTES:
|
|
69
|
+
raise AutError("desktop: source byte cap exceeded")
|
|
70
|
+
try:
|
|
71
|
+
whole = json.loads(raw)
|
|
72
|
+
except (ValueError, UnicodeError):
|
|
73
|
+
whole = None
|
|
74
|
+
body = whole.get("data", whole) if isinstance(whole, dict) else None
|
|
75
|
+
if isinstance(body, dict) and "events" in body:
|
|
76
|
+
if session_id is not None and body.get("session_id", session_id) != session_id:
|
|
77
|
+
raise AutError("desktop: explicit session mismatch")
|
|
78
|
+
if body.get("truncated") is not False:
|
|
79
|
+
coverage["truncated_exports"] += 1
|
|
80
|
+
coverage["source_warnings"] += bool(body.get("warnings"))
|
|
81
|
+
if body.get("total_events") != body.get("returned_events"):
|
|
82
|
+
coverage["filtered_exports"] += 1
|
|
83
|
+
batch = body["events"]
|
|
84
|
+
if not isinstance(batch, list):
|
|
85
|
+
raise AutError("desktop: invalid trace-show events")
|
|
86
|
+
else:
|
|
87
|
+
batch = []
|
|
88
|
+
lines = raw.splitlines()
|
|
89
|
+
for index, line in enumerate(lines):
|
|
90
|
+
try:
|
|
91
|
+
batch.append(json.loads(line))
|
|
92
|
+
except (ValueError, UnicodeError):
|
|
93
|
+
if index == len(lines) - 1 and not raw.endswith(b"\n"):
|
|
94
|
+
coverage["partial_lines"] += 1
|
|
95
|
+
else:
|
|
96
|
+
coverage["malformed_records"] += 1
|
|
97
|
+
for row in batch:
|
|
98
|
+
budget[1] += 1
|
|
99
|
+
if budget[1] > MAX_EVENTS:
|
|
100
|
+
raise AutError("desktop: source event cap exceeded")
|
|
101
|
+
if not isinstance(row, dict):
|
|
102
|
+
raise AutError("desktop: expected object records")
|
|
103
|
+
rows.append(row)
|
|
104
|
+
return rows
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _disposition(response: dict[str, Any]) -> dict[str, Any]:
|
|
108
|
+
payload = response.get("data" if response.get("ok") is True else "error", {})
|
|
109
|
+
if not isinstance(payload, dict):
|
|
110
|
+
payload = {}
|
|
111
|
+
disposition = payload.get("disposition", {})
|
|
112
|
+
if not isinstance(disposition, dict):
|
|
113
|
+
disposition = {}
|
|
114
|
+
delivery = disposition.get("delivery", "unknown")
|
|
115
|
+
retry = disposition.get("retry", "unknown")
|
|
116
|
+
invalid = (
|
|
117
|
+
not isinstance(delivery, str)
|
|
118
|
+
or delivery not in DELIVERY
|
|
119
|
+
or not isinstance(retry, str)
|
|
120
|
+
or retry not in {"safe", "unsafe", "unknown"}
|
|
121
|
+
)
|
|
122
|
+
if invalid:
|
|
123
|
+
delivery, retry = "unknown", "unknown"
|
|
124
|
+
code = payload.get("code")
|
|
125
|
+
return dict(
|
|
126
|
+
delivery=delivery,
|
|
127
|
+
retry=retry,
|
|
128
|
+
disposition_conflict=invalid or DELIVERY[delivery] != retry,
|
|
129
|
+
response_code=code if isinstance(code, str) and code in CODES else None,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def import_desktop(
|
|
134
|
+
store: Store, trace_path: Path, response_path: Path | None, mapping: dict[str, Any]
|
|
135
|
+
) -> dict[str, Any]:
|
|
136
|
+
"""Import bounded native traces with explicit parent association and unknown acceptance."""
|
|
137
|
+
required = {"task_id", "session_id", "parent_call_id", "desktop_session_id"}
|
|
138
|
+
if set(mapping) != required or any(
|
|
139
|
+
not isinstance(value, str) or not value.strip() or len(value) > 512
|
|
140
|
+
for value in mapping.values()
|
|
141
|
+
):
|
|
142
|
+
raise AutError("desktop: four explicit bounded mapping identifiers are required")
|
|
143
|
+
coverage: Counter[str] = Counter()
|
|
144
|
+
budget = [0, 0]
|
|
145
|
+
raw_events = _read(trace_path, coverage, budget, mapping["desktop_session_id"])
|
|
146
|
+
raw_responses = _read(response_path, coverage, budget)
|
|
147
|
+
events: dict[tuple[int, int, int], dict[str, Any]] = {}
|
|
148
|
+
for row in raw_events:
|
|
149
|
+
if row.get("session_id") is not None and row["session_id"] != mapping["desktop_session_id"]:
|
|
150
|
+
raise AutError("desktop: explicit session mismatch")
|
|
151
|
+
key = _identity(row)
|
|
152
|
+
if key in events:
|
|
153
|
+
if events[key] != row:
|
|
154
|
+
raise AutError("desktop: conflicting event copies")
|
|
155
|
+
coverage["duplicate_events"] += 1
|
|
156
|
+
else:
|
|
157
|
+
events[key] = row
|
|
158
|
+
responses: dict[tuple[int, int, int], dict[str, Any]] = {}
|
|
159
|
+
unkeyed_responses: list[dict[str, Any]] = []
|
|
160
|
+
for row in raw_responses:
|
|
161
|
+
if not isinstance(row.get("response"), dict):
|
|
162
|
+
raise AutError("desktop: invalid response envelope")
|
|
163
|
+
identity_fields = ("writer_pid", "writer_proc_start_ms", "command_seq")
|
|
164
|
+
if any(row.get(field) is None for field in identity_fields):
|
|
165
|
+
if any(
|
|
166
|
+
row.get(field) is not None and not _integer(row[field]) for field in identity_fields
|
|
167
|
+
):
|
|
168
|
+
raise AutError("desktop: invalid response identity")
|
|
169
|
+
unkeyed_responses.append(row["response"])
|
|
170
|
+
continue
|
|
171
|
+
key = _identity(row, "command_seq")
|
|
172
|
+
if key in responses and responses[key] != row["response"]:
|
|
173
|
+
raise AutError("desktop: conflicting response copies")
|
|
174
|
+
responses[key] = row["response"]
|
|
175
|
+
writers: dict[tuple[int, int], list[dict[str, Any]]] = defaultdict(list)
|
|
176
|
+
for key, row in sorted(events.items()):
|
|
177
|
+
writers[key[:2]].append(row)
|
|
178
|
+
commands: list[dict[str, Any]] = []
|
|
179
|
+
observations: list[dict[str, Any]] = []
|
|
180
|
+
matched: set[tuple[int, int, int]] = set()
|
|
181
|
+
for writer, rows in writers.items():
|
|
182
|
+
metas = [row for row in rows if row.get("event") == "trace.meta"]
|
|
183
|
+
coverage["missing_meta"] += not metas
|
|
184
|
+
coverage["unknown_schemas"] += sum(meta.get("schema") != 1 for meta in metas)
|
|
185
|
+
seqs = [row["seq"] for row in rows]
|
|
186
|
+
coverage["sequence_gaps"] += sum(b != a + 1 for a, b in zip(seqs, seqs[1:]))
|
|
187
|
+
pending: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
188
|
+
spans: list[tuple[dict[str, Any] | None, dict[str, Any] | None]] = []
|
|
189
|
+
for row in rows:
|
|
190
|
+
name = row.get("command")
|
|
191
|
+
if row.get("event") not in {"command.start", "command.end"}:
|
|
192
|
+
continue
|
|
193
|
+
if not isinstance(name, str) or not name or len(name) > 512:
|
|
194
|
+
raise AutError("desktop: invalid command name")
|
|
195
|
+
if row["event"] == "command.start":
|
|
196
|
+
pending[name].append(row)
|
|
197
|
+
else:
|
|
198
|
+
start = pending[name].pop() if pending[name] else None
|
|
199
|
+
spans.append((start, row))
|
|
200
|
+
coverage["unpaired_ends"] += start is None
|
|
201
|
+
for stack in pending.values():
|
|
202
|
+
for start in stack:
|
|
203
|
+
spans.append((start, None))
|
|
204
|
+
coverage["unpaired_starts"] += 1
|
|
205
|
+
for start, end in spans:
|
|
206
|
+
anchor = start if start is not None else end
|
|
207
|
+
assert anchor is not None
|
|
208
|
+
key = (*writer, anchor["seq"])
|
|
209
|
+
identifier = digest(
|
|
210
|
+
[
|
|
211
|
+
mapping["desktop_session_id"],
|
|
212
|
+
*key,
|
|
213
|
+
"start" if start is not None else "orphan_end",
|
|
214
|
+
]
|
|
215
|
+
)
|
|
216
|
+
result: dict[str, Any] = dict(
|
|
217
|
+
id=identifier,
|
|
218
|
+
command_hash=digest(anchor["command"]),
|
|
219
|
+
status="unknown",
|
|
220
|
+
duration_ms=None,
|
|
221
|
+
delivery="unknown",
|
|
222
|
+
retry="unknown",
|
|
223
|
+
disposition_conflict=False,
|
|
224
|
+
response_code=None,
|
|
225
|
+
trace_ok=end.get("ok") if end is not None and type(end.get("ok")) is bool else None,
|
|
226
|
+
response_ok=None,
|
|
227
|
+
trace_code=end.get("code")
|
|
228
|
+
if end is not None and isinstance(end.get("code"), str) and end.get("code") in CODES
|
|
229
|
+
else None,
|
|
230
|
+
)
|
|
231
|
+
if start is not None and end is not None:
|
|
232
|
+
if type(end.get("ok")) is bool:
|
|
233
|
+
result["status"] = "ok" if end["ok"] else "error"
|
|
234
|
+
if _integer(end.get("duration_ms")):
|
|
235
|
+
result["duration_ms"] = end["duration_ms"]
|
|
236
|
+
response = responses.get(key) if start is not None else None
|
|
237
|
+
if response is not None:
|
|
238
|
+
matched.add(key)
|
|
239
|
+
if (
|
|
240
|
+
response.get("command") != anchor["command"]
|
|
241
|
+
or type(response.get("ok")) is not bool
|
|
242
|
+
):
|
|
243
|
+
coverage["mismatched_responses"] += 1
|
|
244
|
+
else:
|
|
245
|
+
result.update(_disposition(response))
|
|
246
|
+
result["response_ok"] = response["ok"]
|
|
247
|
+
if end is not None and response["ok"] != end.get("ok"):
|
|
248
|
+
result["status"] = "unknown"
|
|
249
|
+
result["disposition_conflict"] = True
|
|
250
|
+
coverage["response_outcome_conflicts"] += 1
|
|
251
|
+
stamp = anchor["ts_ms"]
|
|
252
|
+
if not _integer(stamp):
|
|
253
|
+
raise AutError("desktop: invalid event timestamp")
|
|
254
|
+
try:
|
|
255
|
+
timestamp = datetime.fromtimestamp(int(stamp) / 1000, timezone.utc).isoformat()
|
|
256
|
+
except (ValueError, OverflowError, OSError) as exc:
|
|
257
|
+
raise AutError("desktop: invalid event timestamp") from exc
|
|
258
|
+
commands.append(result)
|
|
259
|
+
if start is None or end is None:
|
|
260
|
+
continue
|
|
261
|
+
observations.append(
|
|
262
|
+
dict(
|
|
263
|
+
version=1,
|
|
264
|
+
kind="tool",
|
|
265
|
+
id=identifier,
|
|
266
|
+
timestamp=timestamp,
|
|
267
|
+
status=result["status"],
|
|
268
|
+
duration_ms=result["duration_ms"],
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
orphan_responses = []
|
|
272
|
+
for response in [
|
|
273
|
+
*unkeyed_responses,
|
|
274
|
+
*(responses[key] for key in sorted(set(responses) - matched)),
|
|
275
|
+
]:
|
|
276
|
+
sanitized = _disposition(response)
|
|
277
|
+
orphan_responses.append(
|
|
278
|
+
dict(
|
|
279
|
+
association="unknown",
|
|
280
|
+
status="ok"
|
|
281
|
+
if response.get("ok") is True
|
|
282
|
+
else "error"
|
|
283
|
+
if response.get("ok") is False
|
|
284
|
+
else "unknown",
|
|
285
|
+
response_code=sanitized["response_code"],
|
|
286
|
+
delivery="unknown",
|
|
287
|
+
retry="unknown",
|
|
288
|
+
)
|
|
289
|
+
)
|
|
290
|
+
coverage["orphan_responses"] = len(orphan_responses)
|
|
291
|
+
coverage["missing_responses"] = len(commands) - len(matched)
|
|
292
|
+
coverage["events"] = len(events)
|
|
293
|
+
coverage["commands"] = len(commands)
|
|
294
|
+
flags = (
|
|
295
|
+
"missing_sources",
|
|
296
|
+
"truncated_exports",
|
|
297
|
+
"source_warnings",
|
|
298
|
+
"filtered_exports",
|
|
299
|
+
"partial_lines",
|
|
300
|
+
"malformed_records",
|
|
301
|
+
"missing_meta",
|
|
302
|
+
"unknown_schemas",
|
|
303
|
+
"sequence_gaps",
|
|
304
|
+
"unpaired_starts",
|
|
305
|
+
"unpaired_ends",
|
|
306
|
+
"mismatched_responses",
|
|
307
|
+
"response_outcome_conflicts",
|
|
308
|
+
"orphan_responses",
|
|
309
|
+
)
|
|
310
|
+
complete = bool(events) and not any(coverage[flag] for flag in flags)
|
|
311
|
+
receipt: dict[str, Any] = dict(
|
|
312
|
+
version=1,
|
|
313
|
+
source="agent-desktop",
|
|
314
|
+
mapping={key: digest(value) for key, value in mapping.items()},
|
|
315
|
+
commands=sorted(commands, key=lambda row: row["id"]),
|
|
316
|
+
orphan_responses=sorted(orphan_responses, key=digest),
|
|
317
|
+
coverage={**dict(coverage), "complete": complete},
|
|
318
|
+
task_acceptance="unknown",
|
|
319
|
+
)
|
|
320
|
+
# Stable stream identity excludes parent calls so children never become parent tool calls.
|
|
321
|
+
ingestion = ingest(
|
|
322
|
+
store.root / "observations.sqlite3",
|
|
323
|
+
observations,
|
|
324
|
+
source="agent-desktop",
|
|
325
|
+
stream="commands",
|
|
326
|
+
role="primary",
|
|
327
|
+
)
|
|
328
|
+
if ingestion["conflicts"]:
|
|
329
|
+
disputed = disputed_tool_ids(
|
|
330
|
+
store.root / "observations.sqlite3", source="agent-desktop", stream="commands"
|
|
331
|
+
)
|
|
332
|
+
for command in commands:
|
|
333
|
+
if command["id"] in disputed:
|
|
334
|
+
command["status"] = "unknown"
|
|
335
|
+
command["disposition_conflict"] = True
|
|
336
|
+
receipt["ingestion"] = ingestion
|
|
337
|
+
receipt["persistence"] = "disputed" if ingestion["conflicts"] else "stored"
|
|
338
|
+
receipt["success"] = not bool(ingestion["conflicts"])
|
|
339
|
+
if ingestion["conflicts"]:
|
|
340
|
+
receipt["coverage"]["complete"] = False
|
|
341
|
+
import_id = store.put("desktop_imports", receipt)
|
|
342
|
+
if ingestion["conflicts"]:
|
|
343
|
+
raise AutError(
|
|
344
|
+
f"desktop: observation conflict; disputed receipt {import_id}; "
|
|
345
|
+
"some observations may already be stored; conflict unresolved"
|
|
346
|
+
)
|
|
347
|
+
return {**receipt, "import_id": import_id}
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""Read local traces without advancing collection or sending their contents."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ..collectors import claude_code, codex
|
|
14
|
+
from ..collectors._util import dict_or_empty
|
|
15
|
+
from ..collectors.activity_common import result_status
|
|
16
|
+
from ..event import parse_ts
|
|
17
|
+
from ..redact import redact
|
|
18
|
+
from ..state import parse_record
|
|
19
|
+
|
|
20
|
+
TEXT_LIMIT = 6000
|
|
21
|
+
BYTE_LIMIT = 16 * 1024 * 1024
|
|
22
|
+
VERSION = 7
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _text(value: object) -> str:
|
|
26
|
+
if isinstance(value, str):
|
|
27
|
+
return value
|
|
28
|
+
if isinstance(value, list):
|
|
29
|
+
if all(isinstance(b, str) for b in value):
|
|
30
|
+
return json.dumps(value, ensure_ascii=False)
|
|
31
|
+
return "\n".join(str(b.get("text", "")) for b in value if isinstance(b, dict))
|
|
32
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _codex_message_text(item: dict[str, Any]) -> str:
|
|
36
|
+
text = _text(item.get("content", item.get("text", "")))
|
|
37
|
+
citation = item.get("memory_citation")
|
|
38
|
+
if not isinstance(citation, dict) or set(citation) != {"entries", "rolloutIds"}:
|
|
39
|
+
return text
|
|
40
|
+
entries, rollouts = citation["entries"], citation["rolloutIds"]
|
|
41
|
+
if not isinstance(entries, list) or not isinstance(rollouts, list) \
|
|
42
|
+
or not all(isinstance(value, str) for value in rollouts) \
|
|
43
|
+
or not (entries or rollouts):
|
|
44
|
+
return text
|
|
45
|
+
lines = []
|
|
46
|
+
for entry in entries:
|
|
47
|
+
if not isinstance(entry, dict) or set(entry) != {"path", "lineStart", "lineEnd", "note"} \
|
|
48
|
+
or not all(isinstance(entry[key], str) for key in ("path", "note")) \
|
|
49
|
+
or type(entry["lineStart"]) is not int or type(entry["lineEnd"]) is not int \
|
|
50
|
+
or not 1 <= entry["lineStart"] <= entry["lineEnd"]:
|
|
51
|
+
return text
|
|
52
|
+
lines.append(f"{entry['path']}:{entry['lineStart']}-{entry['lineEnd']}"
|
|
53
|
+
f"|note=[{entry['note']}]")
|
|
54
|
+
# Native AgentMessage stores citations separately; response_item renders them in the text.
|
|
55
|
+
return (text + "<oai-mem-citation>\n<citation_entries>\n" + "\n".join(lines)
|
|
56
|
+
+ "\n</citation_entries>\n<rollout_ids>\n" + "\n".join(rollouts)
|
|
57
|
+
+ "\n</rollout_ids>\n</oai-mem-citation>")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _recent_records(path: Path, budget: int,
|
|
61
|
+
observation: dict[str, int] | None = None) -> Iterator[tuple[int, int, bytes]]:
|
|
62
|
+
"""Keep the first metadata record and a tail, using byte offsets for unknown ordinals."""
|
|
63
|
+
with path.open("rb") as stream:
|
|
64
|
+
size = stream.seek(0, 2)
|
|
65
|
+
if observation is not None:
|
|
66
|
+
observation["size"] = size
|
|
67
|
+
stream.seek(0)
|
|
68
|
+
if size <= budget:
|
|
69
|
+
offset = 0
|
|
70
|
+
for line in range(1, 1_000_001):
|
|
71
|
+
if offset >= size:
|
|
72
|
+
break
|
|
73
|
+
raw = stream.readline(min(size - offset, budget - offset))
|
|
74
|
+
if not raw:
|
|
75
|
+
break
|
|
76
|
+
yield line, offset, raw
|
|
77
|
+
offset += len(raw)
|
|
78
|
+
return
|
|
79
|
+
header = stream.readline(min(budget, size))
|
|
80
|
+
yield 1, 0, header
|
|
81
|
+
if not header.endswith(b"\n"):
|
|
82
|
+
return
|
|
83
|
+
start = max(len(header), size - (budget - len(header)))
|
|
84
|
+
stream.seek(start)
|
|
85
|
+
tail = stream.read(min(budget - len(header), size - start))
|
|
86
|
+
# The preceding byte is outside the budget, so discard the first fragment.
|
|
87
|
+
end = tail.find(b"\n")
|
|
88
|
+
if end < 0:
|
|
89
|
+
return
|
|
90
|
+
offset = start + end + 1
|
|
91
|
+
for raw in tail[end + 1:].splitlines(keepends=True):
|
|
92
|
+
yield -offset, offset, raw
|
|
93
|
+
offset += len(raw)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def read_session(path: Path, agent: str, max_bytes: int = BYTE_LIMIT,
|
|
97
|
+
since_epoch: float | None = None,
|
|
98
|
+
imports: dict[str, datetime] | None = None, *,
|
|
99
|
+
exclude_automated: bool = False, recent: bool = False) -> dict[str, Any] | None:
|
|
100
|
+
if agent not in {"claude_code", "codex"} or max_bytes < 1:
|
|
101
|
+
raise ValueError("choose a supported trace source and a positive byte limit")
|
|
102
|
+
events: list[dict[str, Any]] = []
|
|
103
|
+
seen: dict[tuple[str, str], str] = {}
|
|
104
|
+
counts: Counter[str] = Counter()
|
|
105
|
+
digest = hashlib.sha256()
|
|
106
|
+
session, cwd, origin = path.stem, "", "unknown"
|
|
107
|
+
last_closed = False
|
|
108
|
+
copied_until = 0
|
|
109
|
+
model: str | None = None
|
|
110
|
+
used = 0
|
|
111
|
+
ranges: list[list[int]] = []
|
|
112
|
+
observation: dict[str, int] = {}
|
|
113
|
+
prefix_omitted = False
|
|
114
|
+
unknown_copy_boundary = False
|
|
115
|
+
|
|
116
|
+
def in_window(record: dict[str, Any]) -> bool:
|
|
117
|
+
if since_epoch is None:
|
|
118
|
+
return True
|
|
119
|
+
stamp = parse_ts(record.get("timestamp"))
|
|
120
|
+
if stamp is None or stamp.tzinfo is None:
|
|
121
|
+
counts["missing_timestamps"] += 1
|
|
122
|
+
return False
|
|
123
|
+
if stamp.timestamp() < since_epoch:
|
|
124
|
+
counts["outside_window"] += 1
|
|
125
|
+
return False
|
|
126
|
+
return True
|
|
127
|
+
|
|
128
|
+
def emit(kind: str, value: object, line: int, record: dict[str, Any], *,
|
|
129
|
+
call: object = "", tool: object = "", status: str = "unknown",
|
|
130
|
+
key: object = "") -> None:
|
|
131
|
+
nonlocal last_closed
|
|
132
|
+
raw = _text(value)
|
|
133
|
+
identity = str(call or key or line)
|
|
134
|
+
signature = hashlib.sha256(json.dumps({"text": raw, "status": status,
|
|
135
|
+
"tool": str(tool or "")}, sort_keys=True).encode()).hexdigest()
|
|
136
|
+
previous = seen.get((kind, identity))
|
|
137
|
+
if previous:
|
|
138
|
+
counts["duplicate_records" if previous == signature else "conflicting_records"] += 1
|
|
139
|
+
return
|
|
140
|
+
seen[kind, identity] = signature
|
|
141
|
+
cleaned = redact(raw)
|
|
142
|
+
clipped = len(cleaned) > TEXT_LIMIT
|
|
143
|
+
counts["truncated_events"] += clipped
|
|
144
|
+
eid = hashlib.sha256(f"{agent}|{session}|{kind}|{identity}".encode()).hexdigest()[:32]
|
|
145
|
+
events.append({"id": eid, "kind": kind, "line": line if line > 0 else None,
|
|
146
|
+
"byte_offset": -line if line < 0 else None,
|
|
147
|
+
"timestamp": record.get("timestamp"), "text": cleaned[:TEXT_LIMIT],
|
|
148
|
+
"truncated": clipped, "source_sha256": signature,
|
|
149
|
+
"call_id": str(call or ""), "tool": str(tool or ""), "status": status})
|
|
150
|
+
if kind in {"call", "prompt"}:
|
|
151
|
+
last_closed = False
|
|
152
|
+
|
|
153
|
+
def prefix_records() -> Iterator[tuple[int, int, bytes]]:
|
|
154
|
+
with path.open("rb") as stream:
|
|
155
|
+
offset = 0
|
|
156
|
+
for line in range(1, 1_000_001):
|
|
157
|
+
raw = stream.readline(max_bytes - offset + 1)
|
|
158
|
+
if not raw:
|
|
159
|
+
return
|
|
160
|
+
yield line, offset, raw
|
|
161
|
+
offset += len(raw)
|
|
162
|
+
|
|
163
|
+
records = _recent_records(path, max_bytes, observation) if recent else prefix_records()
|
|
164
|
+
for line, offset, raw in records:
|
|
165
|
+
prefix_omitted = recent and observation["size"] > max_bytes
|
|
166
|
+
if used + len(raw) > max_bytes:
|
|
167
|
+
counts["byte_cap_reached"] = 1
|
|
168
|
+
break
|
|
169
|
+
if ranges and ranges[-1][1] == offset:
|
|
170
|
+
ranges[-1][1] += len(raw)
|
|
171
|
+
else:
|
|
172
|
+
ranges.append([offset, offset + len(raw)])
|
|
173
|
+
used += len(raw)
|
|
174
|
+
digest.update(raw)
|
|
175
|
+
if not raw.endswith(b"\n"):
|
|
176
|
+
counts["partial_tail"] = 1
|
|
177
|
+
break
|
|
178
|
+
record = parse_record(raw)
|
|
179
|
+
if record is None:
|
|
180
|
+
counts["malformed_records"] += 1
|
|
181
|
+
continue
|
|
182
|
+
kind = record.get("type")
|
|
183
|
+
if agent == "codex":
|
|
184
|
+
payload = dict_or_empty(record.get("payload"))
|
|
185
|
+
if line == 1 and kind == "session_meta":
|
|
186
|
+
session = str(payload.get("id") or payload.get("session_id") or session)
|
|
187
|
+
cwd = str(payload.get("cwd") or "")
|
|
188
|
+
origin = codex.run_origin(payload)
|
|
189
|
+
# Audits discard these sources; avoid normalizing their private bodies first.
|
|
190
|
+
if exclude_automated and origin in {"subagent", "automated"}:
|
|
191
|
+
return None
|
|
192
|
+
if payload.get("forked_from_id"):
|
|
193
|
+
ordinal = payload.get("subagent_history_start_ordinal")
|
|
194
|
+
unknown_copy_boundary = not isinstance(ordinal, int) or ordinal < 1
|
|
195
|
+
copied_until = ordinal if isinstance(ordinal, int) else 0
|
|
196
|
+
if unknown_copy_boundary or (line < 0 and copied_until):
|
|
197
|
+
counts["unknown_copy_boundary"] += 1
|
|
198
|
+
continue
|
|
199
|
+
if 0 < line <= copied_until:
|
|
200
|
+
counts["copied_records"] += 1
|
|
201
|
+
continue
|
|
202
|
+
if kind not in {"session_meta", "turn_context"} \
|
|
203
|
+
and codex.imported_record(record, (imports or {}).get(session)):
|
|
204
|
+
counts["imported_records"] += 1
|
|
205
|
+
continue
|
|
206
|
+
if kind not in {"session_meta", "turn_context"} and not in_window(record):
|
|
207
|
+
continue
|
|
208
|
+
sub = payload.get("type")
|
|
209
|
+
if kind == "turn_context":
|
|
210
|
+
model = payload.get("model") or model
|
|
211
|
+
elif kind == "response_item":
|
|
212
|
+
call_id = payload.get("call_id")
|
|
213
|
+
if sub in {"function_call", "custom_tool_call"}:
|
|
214
|
+
emit("call", payload.get("arguments", payload.get("input", "")),
|
|
215
|
+
line, record, call=call_id, tool=payload.get("name"))
|
|
216
|
+
elif sub in {"function_call_output", "custom_tool_call_output"}:
|
|
217
|
+
value = payload.get("output", "")
|
|
218
|
+
emit("result", value, line, record, call=call_id,
|
|
219
|
+
status=result_status(value))
|
|
220
|
+
elif sub == "message" and payload.get("role") in {"user", "assistant"}:
|
|
221
|
+
emit("prompt" if payload["role"] == "user" else "reply",
|
|
222
|
+
payload.get("content", ""), line, record, key=payload.get("id"))
|
|
223
|
+
elif kind == "event_msg":
|
|
224
|
+
item = dict_or_empty(payload.get("item"))
|
|
225
|
+
if sub == "user_message":
|
|
226
|
+
message = payload.get("message", "")
|
|
227
|
+
if codex.is_scheduled_automation(message):
|
|
228
|
+
origin = "automated"
|
|
229
|
+
emit("prompt", message, line, record)
|
|
230
|
+
elif sub == "agent_message":
|
|
231
|
+
emit("reply", payload.get("message", ""), line, record)
|
|
232
|
+
elif sub in {"task_complete", "turn_aborted"}:
|
|
233
|
+
last_closed = sub == "task_complete"
|
|
234
|
+
emit("boundary", "turn_complete" if last_closed else "turn_aborted",
|
|
235
|
+
line, record)
|
|
236
|
+
elif sub == "item_completed" and item.get("type") == "CommandExecution":
|
|
237
|
+
identity = item.get("id") or f"command-{line}"
|
|
238
|
+
emit("call", item.get("command", ""), line, record,
|
|
239
|
+
call=identity, tool="exec_command")
|
|
240
|
+
emit("result", {"exit_code": item.get("exit_code"),
|
|
241
|
+
"output": item.get("aggregated_output", "")},
|
|
242
|
+
line, record, call=identity,
|
|
243
|
+
status=result_status({"exit_code": item.get("exit_code")}))
|
|
244
|
+
elif sub == "item_completed" and item.get("type") == "McpToolCall":
|
|
245
|
+
identity = item.get("id") or f"mcp-{line}"
|
|
246
|
+
emit("call", item.get("arguments", {}), line, record, call=identity,
|
|
247
|
+
tool=item.get("tool") or item.get("name") or "mcp")
|
|
248
|
+
error = item.get("status") == "failed"
|
|
249
|
+
emit("result", item.get("error") if error else item.get("result", ""),
|
|
250
|
+
line, record, call=identity,
|
|
251
|
+
status="error" if error else "success" if item.get("status")
|
|
252
|
+
== "completed" else "unknown")
|
|
253
|
+
elif sub == "item_completed" and item.get("type") in {"UserMessage", "AgentMessage"}:
|
|
254
|
+
emit("prompt" if item["type"] == "UserMessage" else "reply",
|
|
255
|
+
_codex_message_text(item) if item["type"] == "AgentMessage"
|
|
256
|
+
else item.get("content", item.get("text", "")), line, record,
|
|
257
|
+
key=item.get("id"))
|
|
258
|
+
elif sub == "item_completed" and item.get("type") == "ContextCompaction":
|
|
259
|
+
emit("compaction", "context was compacted", line, record, key=item.get("id"))
|
|
260
|
+
continue
|
|
261
|
+
session = str(record.get("sessionId") or session)
|
|
262
|
+
cwd = str(record.get("cwd") or cwd)
|
|
263
|
+
if record.get("entrypoint"):
|
|
264
|
+
origin = "interactive" if record["entrypoint"] == "cli" else "automated"
|
|
265
|
+
if record.get("isSidechain"):
|
|
266
|
+
origin = "subagent"
|
|
267
|
+
if prefix_omitted and line == 1:
|
|
268
|
+
continue
|
|
269
|
+
if kind not in {"user", "assistant"} and not (
|
|
270
|
+
kind == "system" and record.get("subtype") in {"turn_duration", "compact_boundary"}
|
|
271
|
+
):
|
|
272
|
+
continue
|
|
273
|
+
if not in_window(record):
|
|
274
|
+
continue
|
|
275
|
+
if kind == "system" and record.get("subtype") == "turn_duration":
|
|
276
|
+
last_closed = True
|
|
277
|
+
emit("boundary", "turn_complete", line, record)
|
|
278
|
+
message = dict_or_empty(record.get("message"))
|
|
279
|
+
model = message.get("model") or model
|
|
280
|
+
content = message.get("content", "")
|
|
281
|
+
blocks = [b for b in content if isinstance(b, dict)] if isinstance(content, list) else []
|
|
282
|
+
if kind == "assistant":
|
|
283
|
+
for index, block in enumerate(blocks):
|
|
284
|
+
if block.get("type") == "tool_use":
|
|
285
|
+
emit("call", block.get("input", {}), line, record,
|
|
286
|
+
call=block.get("id"), tool=block.get("name"))
|
|
287
|
+
elif block.get("type") == "text":
|
|
288
|
+
emit("reply", block.get("text", ""), line, record,
|
|
289
|
+
key=f"{record.get('uuid', line)}:{index}")
|
|
290
|
+
elif kind == "user":
|
|
291
|
+
for block in blocks:
|
|
292
|
+
if block.get("type") == "tool_result":
|
|
293
|
+
value = block.get("content", "")
|
|
294
|
+
emit("result", value, line, record, call=block.get("tool_use_id"),
|
|
295
|
+
status=result_status(value, block.get("is_error", False)))
|
|
296
|
+
human = claude_code.human_text(record)
|
|
297
|
+
if human is not None:
|
|
298
|
+
emit("prompt", human, line, record, key=record.get("uuid"))
|
|
299
|
+
if kind == "system" and record.get("subtype") == "compact_boundary":
|
|
300
|
+
emit("compaction", "context was compacted", line, record)
|
|
301
|
+
|
|
302
|
+
if prefix_omitted:
|
|
303
|
+
counts["byte_cap_reached"] = 1
|
|
304
|
+
counts["prefix_omitted"] = 1
|
|
305
|
+
calls = {e["call_id"] for e in events if e["kind"] == "call"}
|
|
306
|
+
results = {e["call_id"] for e in events if e["kind"] == "result"}
|
|
307
|
+
coverage: dict[str, Any] = dict(counts)
|
|
308
|
+
for name in ("partial_tail", "byte_cap_reached"):
|
|
309
|
+
coverage[name] = bool(counts[name])
|
|
310
|
+
coverage.update(calls=len(calls), matched_results=len(calls & results),
|
|
311
|
+
unmatched_calls=len(calls - results), orphan_results=len(results - calls),
|
|
312
|
+
truncated_events=counts["truncated_events"],
|
|
313
|
+
copied_records=counts["copied_records"])
|
|
314
|
+
coverage["complete"] = last_closed and not any(coverage.get(k) for k in (
|
|
315
|
+
"unmatched_calls", "orphan_results", "partial_tail", "byte_cap_reached",
|
|
316
|
+
"truncated_events", "malformed_records", "conflicting_records", "missing_timestamps",
|
|
317
|
+
"unknown_copy_boundary"))
|
|
318
|
+
return {"version": VERSION, "agent": agent, "session_id": session, "cwd": cwd,
|
|
319
|
+
"origin": origin, "model": model, "events": events, "coverage": coverage,
|
|
320
|
+
"source": {"path": str(path), "sha256": digest.hexdigest(), "bytes": used,
|
|
321
|
+
"ranges": ranges,
|
|
322
|
+
"size": observation["size"] if recent else path.stat().st_size}}
|