agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,100 @@
1
+ """Collect explicitly configured normalized SDK events after delivery acknowledgment."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import glob
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import sys
10
+ from typing import Any, Callable, Iterator
11
+
12
+ from ..config import AutError, Config
13
+ from ..event import Batch, UsageEvent, make_event_id
14
+ from ..observations import COUNTS, MAX_BATCH_RECORDS, MAX_FILE_BYTES, MAX_STORE_RECORDS
15
+ from ..sdk import validate_envelope
16
+
17
+ NAME = "generic_sdk"
18
+
19
+
20
+ def describe(cfg: Config, wm: dict[str, Any]) -> str:
21
+ return f"{len(wm.get('records', {}))} acknowledged SDK identities (full scan)"
22
+
23
+
24
+ def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[Batch]:
25
+ pattern = cfg.generic_sdk_paths
26
+ if not pattern or not pattern.strip() or "\x00" in pattern or "://" in pattern:
27
+ raise AutError("generic_sdk_paths requires an explicit local file or glob")
28
+ paths = sorted(set(glob.glob(os.path.expanduser(pattern), recursive=True)))
29
+ if not paths:
30
+ raise AutError("generic_sdk_paths matched no files")
31
+ known = wm.get("records", {})
32
+ pending: dict[str, str] = {}
33
+ events: dict[str, UsageEvent] = {}
34
+ unsupported = unavailable = 0
35
+ for path in paths:
36
+ try:
37
+ with open(path, "rb") as handle:
38
+ data = handle.read(MAX_FILE_BYTES + 1)
39
+ if len(data) > MAX_FILE_BYTES:
40
+ raise ValueError("file exceeds limit")
41
+ # The final unterminated line belongs to the next pass, even if it parses.
42
+ for line in data.split(b"\n")[:-1]:
43
+ value = validate_envelope(json.loads(line))
44
+ native = value.get("native_source")
45
+ if native in cfg.agents:
46
+ raise AutError("generic_sdk overlap: native_source collector is enabled")
47
+ usage = value["usage"]
48
+ identity = make_event_id(NAME, value["producer"], usage["id"])
49
+ digest = hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
50
+ prior = pending.get(identity, known.get(identity))
51
+ if prior is not None:
52
+ if prior != digest:
53
+ raise AutError("generic_sdk conflicting producer/event identity")
54
+ continue
55
+ pending[identity] = digest
56
+ if len(known) + len(pending) > MAX_STORE_RECORDS:
57
+ raise ValueError("identity limit exceeded")
58
+ if any(key in usage for key in ("counter", "temporality", "start")):
59
+ unsupported += 1
60
+ continue
61
+ required = [
62
+ key for key in COUNTS if key not in ("input_tokens_total", "reasoning_tokens")
63
+ ]
64
+ if any(usage[key] is None for key in required):
65
+ unavailable += 1
66
+ continue
67
+ total = sum(usage[key] for key in required)
68
+ if total > 2**63 - 1:
69
+ raise ValueError("total tokens exceeds int64")
70
+ counts = {key: usage[key] for key in (*required, "reasoning_tokens")}
71
+ events[identity] = UsageEvent(
72
+ event_id=identity,
73
+ ts=usage["timestamp"],
74
+ agent=NAME,
75
+ model=value.get("model"),
76
+ provider=value.get("provider"),
77
+ total_tokens=total,
78
+ **counts,
79
+ )
80
+ except (OSError, ValueError, TypeError) as exc:
81
+ raise AutError("generic_sdk malformed or unreadable local source") from exc
82
+ if unsupported or unavailable:
83
+ print(
84
+ f"aut: generic_sdk unsupported={unsupported} unavailable={unavailable}", file=sys.stderr
85
+ )
86
+
87
+ def acknowledge_batch(records: dict[str, str]) -> Callable[[], None]:
88
+ def acknowledge() -> None:
89
+ wm.setdefault("records", {}).update(records)
90
+
91
+ return acknowledge
92
+
93
+ # Validate every source before yielding so late conflicts cannot partially deliver a scan.
94
+ identities = list(pending)
95
+ for offset in range(0, max(1, len(identities)), MAX_BATCH_RECORDS):
96
+ selected = identities[offset : offset + MAX_BATCH_RECORDS]
97
+ yield (
98
+ [events[key] for key in selected if key in events],
99
+ acknowledge_batch({key: pending[key] for key in selected}),
100
+ )
@@ -0,0 +1,397 @@
1
+ """Pi collector: ~/.pi/agent/sessions/**/*.jsonl (pi.dev, @earendil-works/pi-coding-agent)
2
+
3
+ Cleanest of the four: each `message` record carries message.usage
4
+ {input, output, cacheRead, cacheWrite, reasoning, totalTokens, cost} plus message.model. Model
5
+ can also change mid-session via `model_change` records (`modelId`), so we track it as streaming
6
+ state. `compaction` records carry the summary call's own usage. Scoped to the sessions directory
7
+ so unrelated JSONL under ~/.pi cannot pollute totals. Subscription-safe.
8
+
9
+ The session header (line 1: id, timestamp, cwd, parentSession on a fork) rides in the file's
10
+ watermark entry, because a resumed scan never reads line 1 again. `project` is the last part of
11
+ the header cwd (the encoded directory name cannot be decoded: Pi uses dashes for both).
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import glob
17
+ import hashlib
18
+ import json
19
+ import os
20
+ import re
21
+ import time
22
+ from collections import Counter
23
+ from pathlib import Path
24
+ from typing import Any, Iterator, TypedDict
25
+
26
+ from ..config import Config
27
+ from ..event import (Row, ActivityKind, Batch, ActivityEvent, CollectorUnavailable, EventSession,
28
+ UsageEvent, make_event_id)
29
+ from ..repo_keys import RepoKeys
30
+ from ..state import parse_record
31
+ from ..sources import SourceError, engine_homes
32
+ from .activity_common import (IDLE_S, Link, TurnAgg, add_call, best_effort, blank_tool_tally, drop_raw_shas,
33
+ first_record, human_event, ms_between, pad_pending, pop_pending,
34
+ record_commits, text_of, tool_key, turn_attrs)
35
+ from .commits import commit_call, commit_dir, commit_shas
36
+ from ._util import (MALFORMED_RECORD, count_field, dict_or_empty, file_batches, project_name,
37
+ safe_int, str_or_none)
38
+
39
+ NAME = "pi"
40
+ _UUID = re.compile(r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$")
41
+ SKILL = re.compile(r'\s*<skill name="([\w.:-]+)"')
42
+ _ENDS = ("stop", "aborted", "length") # an `error` followed by another assistant is a retry
43
+ # The watermark's format. Older state is upgraded once by _upgrade_state; delete a step from there
44
+ # once no machine can still hold state that old.
45
+ _STATE_V = 2
46
+
47
+
48
+ class FileState(TypedDict, total=False):
49
+ """One file's watermark entry. An entry saved by an older version also holds an unread `dev`."""
50
+ off: int
51
+ ino: int
52
+ model: str | None
53
+ provider: str
54
+ hdr: dict[str, Any] # see header()
55
+ turn: str | None
56
+ act: dict[str, Any] # _Activity's carried state
57
+
58
+
59
+ def sessions_dir(home: Path) -> Path:
60
+ return home / ".pi" / "agent" / "sessions"
61
+
62
+
63
+ def session_files(home: Path) -> list[str]:
64
+ return [path for root in sessions_dirs(home)
65
+ for path in glob.glob(str(root / "**" / "*.jsonl"), recursive=True)]
66
+
67
+
68
+ def sessions_dirs(home: Path) -> list[Path]:
69
+ try:
70
+ return [root / "sessions" for root in engine_homes(home, NAME)]
71
+ except SourceError as exc:
72
+ raise CollectorUnavailable(str(exc)) from exc
73
+
74
+
75
+ def _cost_total(cost: object) -> float | None:
76
+ v = cost.get("total") if isinstance(cost, dict) else cost
77
+ if isinstance(v, bool) or not isinstance(v, (int, float)): # bool is an int subclass
78
+ return None
79
+ try:
80
+ f = float(v)
81
+ except OverflowError: # e.g. a huge int literal
82
+ return None
83
+ if f != f or abs(f) == float("inf"): # drop NaN / inf (the sink numeric can't store them)
84
+ return None
85
+ return f
86
+
87
+
88
+ def uuid_in(path: str) -> str | None:
89
+ """The session uuid in a Pi file name (`<ts>_<uuid>.jsonl`)."""
90
+ m = _UUID.search(os.path.basename(path))
91
+ return m.group(1) if m else None
92
+
93
+
94
+ def header(o: dict[str, Any], path: str) -> dict[str, Any]:
95
+ """What the watermark keeps of line 1. A fork's parent is the header id of the file named in
96
+ parentSession, else the uuid in that name; the path itself is never kept."""
97
+ ps = str_or_none(o.get("parentSession"))
98
+ parent = None
99
+ if ps:
100
+ first = first_record(ps) if ps.endswith(".jsonl") and os.path.isfile(ps) else None
101
+ parent = (str_or_none(first.get("id")) if first and first.get("type") == "session" else None) \
102
+ or uuid_in(ps)
103
+ return {"id": str_or_none(o.get("id")) or uuid_in(path) or Path(path).stem,
104
+ "ts": o.get("timestamp"), "cwd": str_or_none(o.get("cwd")), "parent": parent,
105
+ "fork": bool(ps)}
106
+
107
+
108
+ def is_fork_copy(o: dict[str, Any], hdr: dict[str, Any]) -> bool:
109
+ """A fork copies its parent's entries, with their ids and timestamps, all before its own header."""
110
+ gap = ms_between(o.get("timestamp"), hdr["ts"]) if hdr["fork"] else None
111
+ return gap is not None and gap > 0
112
+
113
+
114
+ class _Activity:
115
+ """Per-file activity state machine; `carried` is the watermark's state (a private copy)."""
116
+
117
+ def __init__(self, carried: dict[str, Any], sid: str, cwd: str | None, keys: RepoKeys) -> None:
118
+ self.carried, self.sid, self.cwd, self.keys = carried, sid, cwd, keys
119
+ self.events: list[ActivityEvent] = []
120
+ self.errors: Counter[str] = Counter() # records whose extraction raised (skipped)
121
+
122
+ def _emit(self, kind: ActivityKind, ts: object, seed: str, turn: str | None, attrs: dict[str, Any]) -> None:
123
+ self.events.append(ActivityEvent(
124
+ event_id=make_event_id(NAME, kind, f"{self.sid}|{seed}"), ts=str_or_none(ts), agent=NAME,
125
+ kind=kind, session_id=self.sid, turn_id=turn, attrs=attrs))
126
+
127
+ def _snapshot(self, end: object) -> None:
128
+ """Publish the same keyed turn again only when its source coverage or aggregates change."""
129
+ turn = self.carried["open"]
130
+ revision = {"revision_sources": sorted(turn["sources"])} if turn.get("sources") else {}
131
+ attrs = {
132
+ "duration_ms": ms_between(turn["ts"], end), "message_count": turn["n"],
133
+ **turn_attrs(self.carried.get("agg") or blank_tool_tally()), **revision}
134
+ link = Link(self.keys, self.carried.setdefault("link", {}))
135
+ link.place(3, self.keys.of_cwd(self.cwd))
136
+ link_attrs = {**link.attrs("interactive"), **revision}
137
+ fingerprint = hashlib.sha256(json.dumps([attrs, link_attrs], sort_keys=True).encode()).hexdigest()
138
+ if turn.get("published") != fingerprint:
139
+ # A stable start timestamp lets idle and terminal snapshots address the same sink row.
140
+ self._emit("turn", turn["ts"], turn["id"], turn["id"], attrs)
141
+ self._emit("link", turn["ts"], turn["id"], turn["id"], link_attrs)
142
+ turn["published"] = fingerprint
143
+
144
+ def _close(self, end: object) -> None:
145
+ self._snapshot(end)
146
+ self.carried.pop("open")
147
+ self.carried.pop("link", None)
148
+ self.carried.pop("agg", None)
149
+ self.carried.pop("pend", None) # a commit call unanswered at turn end never will be
150
+
151
+ def record(self, o: dict[str, Any], sess: EventSession) -> None:
152
+ carried, ts, eid = self.carried, o.get("timestamp"), o.get("id")
153
+ if o.get("type") == "compaction":
154
+ tb = o.get("tokensBefore")
155
+ self._emit("compaction", ts, str(eid), sess.turn_id,
156
+ {"tokens_before": tb} if isinstance(tb, int) else {})
157
+ return
158
+ m = o.get("message")
159
+ if o.get("type") != "message" or not isinstance(m, dict):
160
+ return
161
+ role, content = m.get("role"), m.get("content")
162
+ open_ = carried.get("open")
163
+ link = Link(self.keys, carried.setdefault("link", {}))
164
+ if role != "user": # a prompt's text belongs to the turn it opens
165
+ link.note_text(text_of(content))
166
+ if role == "user":
167
+ self._prompt(ts, eid, content, sess, open_)
168
+ open_ = carried["open"]
169
+ if open_ and (source := self.keys.of(
170
+ f"pi:source:{self.sid}:{json.dumps(o, sort_keys=True)}")):
171
+ sources = open_.setdefault("sources", [])
172
+ if source not in sources:
173
+ sources.append(source)
174
+ if role in ("assistant", "toolResult"):
175
+ if open_:
176
+ open_["n"] += 1
177
+ open_["last"] = ts
178
+ agg = carried.setdefault("agg", blank_tool_tally())
179
+ if role == "assistant":
180
+ self._assistant(m, ts, eid, sess, agg, open_)
181
+ else:
182
+ self._tool_result(m, ts, sess, agg, link)
183
+ carried["prev"] = role
184
+
185
+ def _prompt(self, ts: object, eid: object, content: object, sess: EventSession,
186
+ open_: dict[str, Any] | None) -> None:
187
+ carried = self.carried
188
+ if open_:
189
+ self._close(open_["last"])
190
+ text = text_of(content)
191
+ skill = SKILL.match(text)
192
+ via = "skill" if text.lstrip().startswith('<skill name="') else \
193
+ "queued" if carried.get("prev") == "toolResult" else "typed"
194
+ self._emit("steer", ts, str(eid), sess.turn_id, {
195
+ "via": via, "words": len(text.split()), "chars": len(text),
196
+ "after_interrupt": bool(carried.pop("intr", False))})
197
+ if skill:
198
+ self._emit("invoke", ts, str(eid), sess.turn_id, {"skill": tool_key(skill[1])})
199
+ carried["open"] = {"id": eid, "ts": ts, "n": 1, "last": ts}
200
+ Link(self.keys, carried.setdefault("link", {})).note_text(text)
201
+ carried["agg"] = blank_tool_tally()
202
+
203
+ def _assistant(self, m: dict[str, Any], ts: object, eid: object, sess: EventSession, agg: TurnAgg,
204
+ open_: dict[str, Any] | None) -> None:
205
+ carried, content = self.carried, m.get("content")
206
+ for b in content if isinstance(content, list) else ():
207
+ if isinstance(b, dict) and b.get("type") == "toolCall" and isinstance(b.get("name"), str):
208
+ self._tool_call(b, agg)
209
+ stop = m.get("stopReason")
210
+ carried["intr"] = stop == "aborted"
211
+ if stop == "aborted":
212
+ self._emit("interrupt", ts, str(eid), sess.turn_id,
213
+ {"during_tool": carried.get("prev") == "toolResult"})
214
+ if stop in _ENDS and open_:
215
+ self._close(ts)
216
+
217
+ def _tool_call(self, b: dict[str, Any], agg: TurnAgg) -> None:
218
+ args = b.get("arguments")
219
+ add_call(agg, b["name"], args)
220
+ cmd = args.get("command") if isinstance(args, dict) else None
221
+ if b["name"] == "bash" and isinstance(cmd, str) and isinstance(b.get("id"), str) \
222
+ and (call := commit_call(cmd)):
223
+ self.carried.setdefault("pend", {})[b["id"]] = [*call, commit_dir(cmd)]
224
+
225
+ def _tool_result(self, m: dict[str, Any], ts: object, sess: EventSession, agg: TurnAgg,
226
+ link: Link) -> None:
227
+ err = m.get("isError") is True
228
+ agg["errors"] += err
229
+ tid = m.get("toolCallId")
230
+ pend = self.carried.get("pend") or {}
231
+ if isinstance(tid, str) and tid in pend:
232
+ amend, before, target = pop_pending(pend, tid)
233
+ text = text_of(m.get("content"))
234
+ found = commit_shas(text, err, before)
235
+ for sha, attrs in record_commits(link, self.cwd, target, text, found, amend=amend):
236
+ self._emit("commit", ts, f"{tid}|{sha}", sess.turn_id, attrs)
237
+
238
+
239
+ def _upgrade_state(wm: dict[str, Any]) -> None:
240
+ """Bring state saved by an older version to the current shape, once: the header kept in the
241
+ entry, pending commit calls with their target, and link SHAs keyed. A reset would re-read the
242
+ file instead. Each step is safe on state already upgraded."""
243
+ if wm.get("state_v") == _STATE_V:
244
+ return
245
+ for path, entry in wm.setdefault("files", {}).items():
246
+ if safe_int(entry.get("off")) and "hdr" not in entry:
247
+ first = first_record(path)
248
+ if first and first.get("type") == "session":
249
+ entry["hdr"] = header(first, path)
250
+ pad_pending(entry.get("act"))
251
+ drop_raw_shas(entry.get("act"))
252
+ wm["state_v"] = _STATE_V
253
+
254
+
255
+ def _note_model_change(o: dict[str, Any], cur: FileState) -> None:
256
+ cur["model"] = str_or_none(o.get("modelId")) or str_or_none(o.get("model")) or cur.get("model")
257
+ provider = str_or_none(o.get("provider"))
258
+ if provider:
259
+ cur["provider"] = provider
260
+
261
+
262
+ def _record_id(o: dict[str, Any], path: str) -> object:
263
+ return o.get("id") or f"{path}:{o.get('timestamp')}"
264
+
265
+
266
+ def _usage_event(o: dict[str, Any], msg: dict[str, Any], cur: FileState, path: str, project: str | None,
267
+ sess: EventSession | None) -> UsageEvent | None:
268
+ """The usage row of a record, None if it has no tokens. Raises on malformed token counts."""
269
+ if o.get("type") == "message":
270
+ u, model, provider = msg.get("usage"), msg.get("model"), msg.get("provider")
271
+ elif o.get("type") == "compaction": # the summary call's own usage; no model on the record
272
+ u, model, provider = o.get("usage"), None, None
273
+ else:
274
+ return None
275
+ if not isinstance(u, dict):
276
+ return None
277
+ inp = count_field(u, "input")
278
+ out = count_field(u, "output")
279
+ cr = count_field(u, "cacheRead")
280
+ cw = count_field(u, "cacheWrite")
281
+ total = int(u.get("totalTokens") or (inp + out + cr + cw))
282
+ # a SUBSET of output: informational, never added to totals or cost
283
+ rea = int(u["reasoning"]) if u.get("reasoning") is not None else None
284
+ if total == 0:
285
+ return None
286
+ return UsageEvent(
287
+ event_id=make_event_id(NAME, _record_id(o, path)),
288
+ ts=o.get("timestamp"),
289
+ agent=NAME,
290
+ model=str_or_none(model) or cur["model"],
291
+ provider=str_or_none(provider) or cur.get("provider"),
292
+ input_tokens=inp,
293
+ output_tokens=out,
294
+ cache_read_tokens=cr,
295
+ cache_write_tokens=cw,
296
+ reasoning_tokens=rea,
297
+ total_tokens=total,
298
+ est_cost_usd=_cost_total(u.get("cost")),
299
+ project=project,
300
+ session=sess,
301
+ )
302
+
303
+
304
+ def _advance_session(o: dict[str, Any], hdr: dict[str, Any], cur: FileState, is_prompt: bool) -> EventSession | None:
305
+ """The record's session, a prompt opening a new turn; None for a fork's copied entry, whose
306
+ usage id dedupes but whose session and activity rows belong to the parent."""
307
+ if is_fork_copy(o, hdr):
308
+ return None
309
+ if is_prompt:
310
+ cur["turn"] = str_or_none(o.get("id")) or cur.get("turn")
311
+ return EventSession(hdr["id"], hdr["parent"], "unknown", cur.get("turn"))
312
+
313
+
314
+ def _chunk_rows(path: str, lines: list[bytes], cur: FileState, keys: RepoKeys,
315
+ skipped: Counter[str]) -> list[Row]:
316
+ """All rows of one file's new lines; updates `cur`, the private copy of its watermark entry."""
317
+ hdr = cur.get("hdr") or header({}, path)
318
+ act = _Activity(cur.setdefault("act", {}), hdr["id"], hdr["cwd"], keys)
319
+ events: list[Row] = []
320
+ for raw in lines:
321
+ o = parse_record(raw)
322
+ if o is None:
323
+ skipped["malformed"] += 1
324
+ continue
325
+ t = o.get("type")
326
+ if t == "session":
327
+ if "hdr" not in cur:
328
+ cur["hdr"] = hdr = header(o, path)
329
+ act.sid, act.cwd = hdr["id"], hdr["cwd"]
330
+ continue
331
+ if t == "model_change":
332
+ _note_model_change(o, cur)
333
+ continue
334
+ msg = dict_or_empty(o.get("message"))
335
+ is_prompt = t == "message" and msg.get("role") == "user"
336
+ sess = _advance_session(o, hdr, cur, is_prompt)
337
+ if sess is not None:
338
+ best_effort(act.errors, act.record, o, sess)
339
+ project = project_name(hdr["cwd"])
340
+ if is_prompt:
341
+ # A human prompt: plain text, no usage block. Emit as a human turn.
342
+ events.append(human_event(NAME, (_record_id(o, path),), o.get("timestamp"), sess, project))
343
+ continue
344
+ try:
345
+ event = _usage_event(o, msg, cur, path, project, sess)
346
+ except MALFORMED_RECORD:
347
+ skipped["malformed"] += 1
348
+ continue
349
+ if event:
350
+ events.append(event)
351
+ skipped.update(act.errors)
352
+ return events + act.events
353
+
354
+
355
+ def _trim_state(cur: FileState) -> None:
356
+ if not cur["act"].get("link"):
357
+ cur["act"].pop("link", None)
358
+ if not cur["act"]:
359
+ del cur["act"]
360
+
361
+
362
+ def _fresh_state(st: os.stat_result) -> FileState:
363
+ return {"off": 0, "ino": st.st_ino, "model": None}
364
+
365
+
366
+ def _idle_snapshot(path: str, cur: FileState, keys: RepoKeys) -> list[ActivityEvent]:
367
+ """An idle EOF publishes a snapshot but keeps pending calls for a resumed session."""
368
+ carried = cur.get("act") or {}
369
+ if not carried.get("open"):
370
+ return []
371
+ try:
372
+ st = os.stat(path)
373
+ except OSError:
374
+ return []
375
+ if cur["off"] < st.st_size or time.time() - st.st_mtime < IDLE_S:
376
+ return []
377
+ hdr = cur.get("hdr") or header({}, path)
378
+ act = _Activity(carried, hdr["id"], hdr["cwd"], keys)
379
+ act._snapshot(carried["open"]["last"])
380
+ return act.events
381
+
382
+
383
+ def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[Batch]:
384
+ if not any(root.exists() for root in sessions_dirs(cfg.home)):
385
+ return
386
+ _upgrade_state(wm)
387
+ keys = RepoKeys(cfg.repo_key_salt)
388
+
389
+ def chunk_rows(path: str, cur: FileState, _off: int, lines: list[bytes],
390
+ skipped: Counter[str]) -> list[Row]:
391
+ events = _chunk_rows(path, lines, cur, keys, skipped)
392
+ events += _idle_snapshot(path, cur, keys)
393
+ _trim_state(cur)
394
+ return events
395
+
396
+ yield from file_batches(NAME, session_files(cfg.home), wm["files"], _fresh_state, chunk_rows,
397
+ revisit=lambda cur: bool((cur.get("act") or {}).get("open")))