agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,612 @@
1
+ """Codex CLI collector: ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl
2
+
3
+ Streaming parse: session_meta gives provider + cwd, turn_context updates the current model,
4
+ and each `token_count` event carries a per-turn `last_token_usage` delta. We emit one row per
5
+ token_count event (its delta), NOT the cumulative total, so re-scans never double-count.
6
+ Codex's input_tokens INCLUDES cached, so we split it into fresh input + cache_read to match
7
+ the other agents. Subscription-safe (plan_type: pro observed).
8
+
9
+ Idempotency: the event id is a hash of stable content (session, timestamp, and the token
10
+ fields), NOT a positional ordinal, so a duplicated or reordered token_count line dedups at
11
+ the sink instead of double-counting. Per-file streaming state (model/session/offset/inode) is
12
+ persisted so resuming from a byte offset keeps the right attribution across scans.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import glob
18
+ import os
19
+ from collections import Counter
20
+ from datetime import datetime, timezone
21
+ from pathlib import Path
22
+ from typing import Any, Iterator, NamedTuple, TypedDict
23
+
24
+ from ..config import Config
25
+ from ..diagnostics import observe_client
26
+ from ..event import (Row, ActivityKind, Batch, ActivityEvent, CollectorUnavailable, EventSession,
27
+ RunOrigin, UsageEvent, make_event_id, parse_ts)
28
+ from ..repo_keys import RepoKeys, repo_identity
29
+ from ..state import parse_record
30
+ from ..sources import SourceError, _read_json, engine_homes
31
+ from .activity_common import (Link, TurnAgg, add_call, best_effort, blank_tool_tally, drop_raw_shas,
32
+ first_record, human_event, ms_between, record_commits, text_of,
33
+ result_status, turn_attrs)
34
+ from .commits import commit_call, commit_dir, commit_shas
35
+ from ._util import (MALFORMED_RECORD, count_field, dict_or_empty, file_batches, project_name,
36
+ safe_int, str_or_none)
37
+
38
+ NAME = "codex"
39
+ # The watermark's format. Older state is upgraded once by _upgrade_state; delete a step from there
40
+ # once no machine can still hold state that old.
41
+ _STATE_V = 2
42
+
43
+
44
+ class FileState(TypedDict, total=False):
45
+ """One file's watermark entry. An entry saved by an older version also holds an unread `dev`."""
46
+ off: int
47
+ ino: int
48
+ model: str | None
49
+ provider: str | None
50
+ session_id: str | None # the usage event_id seed; see _note_session_meta
51
+ cwd: str | None
52
+ tc_ord: int # token_count records so far
53
+ um_ord: int # kept user messages so far
54
+ meta: dict[str, Any] # identity from the first session_meta; see _set_meta
55
+ turn: str | None
56
+ has_um: bool # the file has user_message records
57
+ skip: int # copied lines of a fork still unread; see parsed_records
58
+ act: dict[str, Any] # _Activity's carried state
59
+ import_checked: str # import boundary used to rebuild carried state
60
+ import_has_um: bool # imported prefix format, independent of the later native format
61
+
62
+
63
+ def sessions_dir(home: Path) -> Path:
64
+ return home / ".codex" / "sessions"
65
+
66
+
67
+ def session_files(home: Path) -> list[str]:
68
+ # Codex moves finished rollouts to archived_sessions; event ids are content-based, so a moved
69
+ # file re-read under its new path re-emits the same ids and the sink dedups them.
70
+ # Plain .jsonl only: the .jsonl.zst archives are from older Codex builds (last seen 2026-04).
71
+ return [path for root in sessions_dirs(home)
72
+ for pattern in (root / "**" / "rollout-*.jsonl",
73
+ root.parent / "archived_sessions" / "rollout-*.jsonl")
74
+ for path in glob.glob(str(pattern), recursive=True)]
75
+
76
+
77
+ def sessions_dirs(home: Path) -> list[Path]:
78
+ try:
79
+ return [root / "sessions" for root in engine_homes(home, NAME)]
80
+ except SourceError as exc:
81
+ raise CollectorUnavailable(str(exc)) from exc
82
+
83
+
84
+ def imported_until(home: Path) -> dict[str, datetime]:
85
+ """Imported history has import-time timestamps, not new native work timestamps."""
86
+ out: dict[str, datetime] = {}
87
+ for root in sessions_dirs(home):
88
+ for sid, bound in _imported_until(root.parent).items():
89
+ out[sid] = max(out.get(sid, bound), bound)
90
+ return out
91
+
92
+
93
+ def _imported_until(root: Path) -> dict[str, datetime]:
94
+ try:
95
+ value = _read_json(root / "external_agent_session_imports.json")
96
+ if not isinstance(value, dict) or not isinstance(value.get("records"), list):
97
+ raise ValueError("Invalid import index")
98
+ out: dict[str, datetime] = {}
99
+ for record in value["records"]:
100
+ if (not isinstance(record, dict)
101
+ or not isinstance(record.get("imported_thread_id"), str)
102
+ or not record["imported_thread_id"]
103
+ or type(record.get("imported_at")) is not int or record["imported_at"] < 0):
104
+ raise ValueError("Invalid import record")
105
+ # shortcut: second precision omits same-second work, use offsets when Codex records them.
106
+ bound = datetime.fromtimestamp(record["imported_at"] + 1, timezone.utc)
107
+ sid = record["imported_thread_id"]
108
+ out[sid] = max(out.get(sid, bound), bound)
109
+ return out
110
+ except FileNotFoundError:
111
+ return {}
112
+ except (OSError, ValueError, OverflowError, RecursionError) as exc:
113
+ raise CollectorUnavailable("codex: import index is unreadable or invalid") from exc
114
+
115
+
116
+ def imported_record(record: dict[str, Any], bound: datetime | None) -> bool:
117
+ if bound is None:
118
+ return False
119
+ stamp = parse_ts(record.get("timestamp"))
120
+ return stamp is None or stamp.tzinfo is None or stamp < bound
121
+
122
+
123
+ def thread_names(home: Path) -> dict[str, str]:
124
+ """session id -> the name Codex gave the thread (session_index.jsonl). Best effort."""
125
+ names: dict[str, str] = {}
126
+ for root in sessions_dirs(home):
127
+ try:
128
+ with open(root.parent / "session_index.jsonl", encoding="utf-8") as fh:
129
+ for line in fh:
130
+ o = parse_record(line)
131
+ if o and isinstance(o.get("id"), str) and isinstance(o.get("thread_name"), str):
132
+ names[o["id"]] = o["thread_name"]
133
+ except (OSError, ValueError): # no index, or a bad byte: keep the names read so far
134
+ continue
135
+ return names
136
+
137
+
138
+ def _fresh_state(st: os.stat_result) -> FileState:
139
+ return {"off": 0, "ino": st.st_ino, "model": None, "provider": "openai",
140
+ "session_id": None, "cwd": None, "tc_ord": 0, "um_ord": 0}
141
+
142
+
143
+ # session_meta.source values that mean the thread was started by software, not by a person:
144
+ # 'exec' is a scripted `codex exec`, 'mcp' is a programmatic MCP call, and a dict carrying
145
+ # 'subagent' is a thread another agent spawned (whose opening prompt is the parent agent
146
+ # talking, and which would double-count the human prompt that started the parent anyway).
147
+ _PROGRAM_SOURCES = {"exec", "mcp"}
148
+
149
+
150
+ def run_origin(p: dict[str, Any]) -> RunOrigin:
151
+ """run_origin from the thread's FIRST session_meta. Structural signals only: originator,
152
+ source and thread_source are recorded by codex itself.
153
+
154
+ Prompts of a subagent or automated thread are not human turns. Without this every
155
+ user_message counts as 'a prompt you sent', which on a machine that runs scheduled
156
+ automations, `codex exec` harnesses and agent-spawned subagents overstates human input
157
+ several-fold, and since the showcase reports agent_turns_per_human_turn, an inflated
158
+ denominator silently understates leverage."""
159
+ src, orig = p.get("source"), p.get("originator")
160
+ if isinstance(src, dict):
161
+ return "subagent" if "subagent" in src else "unknown"
162
+ if (src in _PROGRAM_SOURCES or p.get("thread_source") == "automation"
163
+ # another coding agent driving codex through its own integration (rescue/review)
164
+ or (isinstance(orig, str) and orig.strip().lower() == "claude code")):
165
+ return "automated"
166
+ return "interactive" if src in ("vscode", "cli") else "unknown"
167
+
168
+
169
+ def is_scheduled_automation(message: object) -> bool:
170
+ """True for the preamble older Codex Desktop files put on a scheduled automation run (newer
171
+ files flag it structurally with thread_source). Both markers are required so an ordinary
172
+ message that merely opens with the word 'Automation' is not swallowed."""
173
+ if not isinstance(message, str):
174
+ return False
175
+ head = message.lstrip()[:200]
176
+ return head.startswith("Automation:") and "Automation ID:" in head
177
+
178
+
179
+ def _set_meta(cur: FileState, p: dict[str, Any], keys: RepoKeys) -> None:
180
+ """Identity, origin and cwd from the FIRST session_meta only: a forked file repeats its
181
+ parent's meta on a later line. Its git block gives the link fields every turn carries: the
182
+ repo from the cwd, else the remote URL (hashed, never kept), and the branch, as keys."""
183
+ observe_client(p.get("originator"))
184
+ cur["meta"] = {"id": str_or_none(p.get("id")), "parent": str_or_none(p.get("parent_thread_id")),
185
+ "origin": run_origin(p)}
186
+ cur["cwd"] = p.get("cwd") or cur.get("cwd")
187
+ cur["meta"]["link"] = _meta_link(p, keys)
188
+
189
+
190
+ def _meta_link(p: dict[str, Any], keys: RepoKeys) -> dict[str, Any]:
191
+ g = dict_or_empty(p.get("git"))
192
+ rk = keys.of_cwd(p.get("cwd"))
193
+ if not rk and isinstance(g.get("repository_url"), str):
194
+ try:
195
+ rk = keys.of(repo_identity(g["repository_url"]))
196
+ except ValueError: # a malformed URL names no repo: the key stays NULL
197
+ rk = None
198
+ # shortcut: a later turn_context cwd in another repo keeps the first meta's repo; take the
199
+ # turn's cwd if Codex threads start moving between repos
200
+ link = Link(keys, {})
201
+ link.place(2, rk, g.get("branch"))
202
+ return link.s
203
+
204
+
205
+ def _session(cur: FileState, path: str) -> EventSession:
206
+ m = cur.get("meta") or {}
207
+ return EventSession(m.get("id") or cur.get("session_id") or os.path.basename(path),
208
+ m.get("parent"), m.get("origin") or "unknown", cur.get("turn"))
209
+
210
+
211
+ def script_of(cmd: object) -> str | None:
212
+ """The shell script of a CommandExecution: codex runs `[shell, -c|-lc, script]`."""
213
+ if isinstance(cmd, str):
214
+ return cmd
215
+ if isinstance(cmd, list) and all(isinstance(x, str) for x in cmd):
216
+ return cmd[2] if len(cmd) == 3 and cmd[1] in ("-c", "-lc") else " ".join(cmd)
217
+ return None
218
+
219
+
220
+ def prompt_of(rtype: object, payload: dict[str, Any], has_um: bool) -> tuple[str, int] | None:
221
+ """(text, images) when the record is a prompt: a user_message, or a UserMessage item in a
222
+ file that has no user_message (older files carry both for one prompt)."""
223
+ if rtype != "event_msg":
224
+ return None
225
+ if payload.get("type") == "user_message":
226
+ imgs = payload.get("local_images")
227
+ msg = payload.get("message")
228
+ return (msg if isinstance(msg, str) else ""), len(imgs) if isinstance(imgs, list) else 0
229
+ it = payload.get("item")
230
+ if payload.get("type") == "item_completed" and isinstance(it, dict) \
231
+ and it.get("type") == "UserMessage" and not has_um:
232
+ content = it.get("content")
233
+ blocks = [b for b in content if isinstance(b, dict)] if isinstance(content, list) else []
234
+ return text_of(content), sum(b.get("type") != "text" for b in blocks)
235
+ return None
236
+
237
+
238
+ class _Activity:
239
+ """Per-file activity extractor over `cur`, the private copy of the file's watermark entry."""
240
+
241
+ # the record being read, set by record()
242
+ sess: EventSession
243
+ link: Link
244
+ agg: TurnAgg
245
+
246
+ def __init__(self, cur: FileState, path: str, keys: RepoKeys) -> None:
247
+ self.cur, self.path, self.keys = cur, path, keys
248
+ self.carried: dict[str, Any] = cur.setdefault("act", {})
249
+ self.events: list[ActivityEvent] = []
250
+ self.errors: Counter[str] = Counter() # records whose extraction raised (skipped)
251
+
252
+ def _emit(self, kind: ActivityKind, ts: object, seed: str, attrs: dict[str, Any]) -> None:
253
+ sess = self.sess
254
+ self.events.append(ActivityEvent(
255
+ event_id=make_event_id(NAME, kind, f"{sess.session_id}|{seed}"), ts=str_or_none(ts),
256
+ agent=NAME, kind=kind, session_id=sess.session_id, turn_id=sess.turn_id, attrs=attrs))
257
+
258
+ def record(self, o: dict[str, Any]) -> None:
259
+ """Non-prompt activity of one record: tool aggregates, interrupt, turn, commit, compaction,
260
+ link."""
261
+ record_type, payload = o.get("type"), dict_or_empty(o.get("payload"))
262
+ ts = o.get("timestamp")
263
+ self.sess = _session(self.cur, self.path)
264
+ self.link = Link(self.keys, self.carried.setdefault("link", {}))
265
+ self.agg = self.carried.setdefault("agg", blank_tool_tally())
266
+ if record_type == "response_item":
267
+ self.link.note_text(payload.get("output")) # a tool call's output
268
+ self._tool_call(payload)
269
+ elif record_type == "event_msg":
270
+ self._event_msg(payload, ts)
271
+
272
+ def _event_msg(self, payload: dict[str, Any], ts: object) -> None:
273
+ kind, item = payload.get("type"), payload.get("item")
274
+ turn_seed = str(payload.get("turn_id") or ts)
275
+ if kind in ("task_complete", "turn_aborted"):
276
+ self._close_link(ts, turn_seed)
277
+ if kind == "agent_message":
278
+ self.link.note_text(payload.get("message"))
279
+ elif kind == "task_started":
280
+ self.carried["agg"], self.carried["start"] = blank_tool_tally(), ts
281
+ elif kind == "turn_aborted" and payload.get("reason") == "interrupted":
282
+ self.carried["intr"] = True
283
+ self._emit("interrupt", ts, turn_seed, {})
284
+ elif kind == "task_complete":
285
+ self._turn(payload.get("duration_ms"), ts, turn_seed)
286
+ elif kind == "item_completed" and isinstance(item, dict):
287
+ self._item(item, ts)
288
+
289
+ def _close_link(self, ts: object, seed: str) -> None:
290
+ self.link.merge((self.cur.get("meta") or {}).get("link") or {})
291
+ self._emit("link", ts, seed, self.link.attrs(self.sess.run_origin))
292
+ self.link.reset()
293
+
294
+ def _turn(self, d: object, ts: object, seed: str) -> None:
295
+ dur = d if isinstance(d, int) and not isinstance(d, bool) \
296
+ else ms_between(self.carried.get("start"), ts)
297
+ self._emit("turn", ts, seed, {"duration_ms": dur, **turn_attrs(self.agg)})
298
+ self.carried["agg"] = blank_tool_tally()
299
+ self.carried.pop("start", None)
300
+
301
+ def _tool_call(self, payload: dict[str, Any]) -> None:
302
+ if payload.get("type") in {"function_call_output", "custom_tool_call_output"}:
303
+ self._count_error(payload.get("call_id"), result_status(payload.get("output")) == "error")
304
+ return
305
+ name = payload.get("name")
306
+ if payload.get("type") in ("function_call", "custom_tool_call") and isinstance(name, str):
307
+ # an MCP namespace is a server label (free text); other namespaces are codex's own.
308
+ # Older CLI files name MCP tools `<server>__<tool>` with no namespace; codex's own
309
+ # tool names never contain `__`.
310
+ ns = payload.get("namespace")
311
+ mcp = (isinstance(ns, str) and ns.startswith("mcp__")) or "__" in name
312
+ if mcp: # Desktop may log the same call again as an McpToolCall item
313
+ self.agg.setdefault("mcp_ids", []).append(payload.get("call_id"))
314
+ add_call(self.agg, "mcp" if mcp else name, payload.get("arguments", payload.get("input")))
315
+
316
+ def _count_error(self, call_id: object, failed: bool) -> None:
317
+ if not failed:
318
+ return
319
+ if isinstance(call_id, str):
320
+ ids = self.agg.setdefault("error_ids", [])
321
+ if call_id in ids:
322
+ return
323
+ ids.append(call_id)
324
+ self.agg["errors"] += 1
325
+
326
+ def _item(self, item: dict[str, Any], ts: object) -> None:
327
+ typ = item.get("type")
328
+ if typ == "CommandExecution":
329
+ code = item.get("exit_code")
330
+ err = isinstance(code, int) and code != 0
331
+ self._count_error(item.get("id"), err)
332
+ script = script_of(item.get("command"))
333
+ call = commit_call(script) if script else None
334
+ out_text = item.get("aggregated_output")
335
+ out_text = out_text if isinstance(out_text, str) else ""
336
+ self.link.note_text(out_text)
337
+ if call and script:
338
+ found = commit_shas(out_text, err, call[1])
339
+ for sha, attrs in record_commits(self.link, item.get("cwd"), commit_dir(script),
340
+ out_text, found, amend=call[0]):
341
+ self._emit("commit", ts, f"{item.get('id')}|{sha}", attrs)
342
+ elif typ == "McpToolCall":
343
+ # Desktop logs most MCP calls only as this item; one that pairs (same id) with an
344
+ # MCP function_call of the turn was already counted
345
+ if item.get("id") not in self.agg.get("mcp_ids", ()):
346
+ add_call(self.agg, "mcp", item.get("arguments"))
347
+ self._count_error(item.get("id"), item.get("status") == "failed")
348
+ elif typ == "ContextCompaction":
349
+ self._emit("compaction", ts, str(item.get("id") or ts), {})
350
+
351
+
352
+ def _upgrade_state(wm: dict[str, Any], keys: RepoKeys) -> None:
353
+ """Bring state saved by an older version to the current shape, once. An entry from before
354
+ um_ord existed is dropped, so the file is re-read from 0 and user_message ordinals (and thus
355
+ their event_ids) stay consistent if it is later reset and re-read; that is safe because
356
+ event_id is content-deterministic and re-emitted rows dedup at the sink. An entry from before
357
+ identity (or its link fields) was kept takes them from the file's first record. Link SHAs
358
+ saved before they were keyed are dropped. Each step is safe on state already upgraded."""
359
+ if wm.get("state_v") == _STATE_V:
360
+ return
361
+ files = wm.setdefault("files", {})
362
+ for path, entry in list(files.items()):
363
+ if "um_ord" not in entry:
364
+ del files[path]
365
+ continue
366
+ drop_raw_shas(entry.get("act"))
367
+ if safe_int(entry.get("off")) and "link" not in (entry.get("meta") or {}):
368
+ first = first_record(path)
369
+ if first and first.get("type") == "session_meta" \
370
+ and isinstance(first.get("payload"), dict):
371
+ if entry.get("meta"): # identity stays as it was first read
372
+ entry["meta"]["link"] = _meta_link(first["payload"], keys)
373
+ else:
374
+ _set_meta(entry, first["payload"], keys)
375
+ wm["state_v"] = _STATE_V
376
+
377
+
378
+ class ParsedRecord(NamedTuple):
379
+ record: dict[str, Any]
380
+ type: object
381
+ payload: dict[str, Any]
382
+ copied: bool
383
+
384
+
385
+ def parsed_records(lines: list[bytes], off: int, cur: FileState,
386
+ skipped: Counter[str]) -> list[ParsedRecord]:
387
+ """Each new line that has a payload. A forked child's file
388
+ opens with a copy of its parent's history: lines before the first meta's
389
+ subagent_history_start_ordinal (0-based). `skip` is how many copied lines are still unread.
390
+ Only a fork: a spawned child can carry an ordinal too, with no copy."""
391
+ recs = []
392
+ skip = cur.pop("skip", 0)
393
+ for i, raw in enumerate(lines):
394
+ copied, skip = skip > 0, max(skip - 1, 0)
395
+ o = parse_record(raw)
396
+ if o is None:
397
+ skipped["malformed"] += 1
398
+ continue
399
+ p = o.get("payload")
400
+ if isinstance(p, dict):
401
+ if not off and not i and o.get("type") == "session_meta" and p.get("forked_from_id"):
402
+ skip = max(safe_int(p.get("subagent_history_start_ordinal")) - 1, 0)
403
+ copied = True
404
+ recs.append(ParsedRecord(o, o.get("type"), p, copied))
405
+ if skip:
406
+ cur["skip"] = skip
407
+ return recs
408
+
409
+
410
+ def _note_session_meta(payload: dict[str, Any], cur: FileState, keys: RepoKeys) -> None:
411
+ # the usage event_id seed keeps its old rule (payload.session_id, any meta line),
412
+ # so stored usage rows never change id; session rows use the thread id instead
413
+ cur["session_id"] = payload.get("session_id") or payload.get("id") or cur["session_id"]
414
+ cur["provider"] = str_or_none(payload.get("model_provider")) or cur["provider"]
415
+ if "meta" not in cur:
416
+ _set_meta(cur, payload, keys)
417
+
418
+
419
+ def _note_context(rtype: object, payload: dict[str, Any], copied: bool, cur: FileState, keys: RepoKeys) -> None:
420
+ """Streaming state from a record: session identity, model, cwd, the current turn."""
421
+ if rtype == "session_meta":
422
+ _note_session_meta(payload, cur, keys)
423
+ elif rtype == "turn_context":
424
+ cur["model"] = str_or_none(payload.get("model")) or cur["model"]
425
+ if payload.get("cwd"):
426
+ cur["cwd"] = payload.get("cwd")
427
+ elif rtype == "event_msg" and payload.get("type") == "task_started" and not copied:
428
+ cur["turn"] = str_or_none(payload.get("turn_id")) or cur.get("turn")
429
+
430
+
431
+ def _token_event(o: dict[str, Any], payload: dict[str, Any], cur: FileState, path: str) -> UsageEvent | None:
432
+ """The usage row of a token_count record, None if it has no tokens. Raises on malformed counts."""
433
+ info = payload.get("info")
434
+ last = info.get("last_token_usage") if isinstance(info, dict) else None
435
+ if not isinstance(last, dict):
436
+ return None
437
+ inp = count_field(last, "input_tokens")
438
+ cached = count_field(last, "cached_input_tokens")
439
+ out = count_field(last, "output_tokens")
440
+ rea = count_field(last, "reasoning_output_tokens")
441
+ total = int(last.get("total_tokens") or (inp + out))
442
+ cur["tc_ord"] += 1
443
+ # Known gap: some Desktop rows carry only total_tokens (input and output 0) and
444
+ # no model; they are dropped here. See CONTRACT.md.
445
+ if inp + out + rea == 0:
446
+ return None
447
+ ts = o.get("timestamp")
448
+ sid = cur["session_id"] or os.path.basename(path)
449
+ # content hash, not position; only fall back to the ordinal if ts is absent.
450
+ tiebreak = "" if ts else cur["tc_ord"]
451
+ return UsageEvent(
452
+ event_id=make_event_id(NAME, sid, ts or "", inp, cached, out, rea, total, tiebreak),
453
+ ts=ts,
454
+ agent=NAME,
455
+ model=cur["model"],
456
+ provider=cur["provider"] or "openai",
457
+ input_tokens=max(inp - cached, 0),
458
+ output_tokens=out,
459
+ cache_read_tokens=cached,
460
+ cache_write_tokens=0,
461
+ reasoning_tokens=rea,
462
+ total_tokens=total,
463
+ project=project_name(cur["cwd"]),
464
+ session=_session(cur, path),
465
+ )
466
+
467
+
468
+ def _prompt_rows(o: dict[str, Any], prompt: tuple[str, int], cur: FileState, path: str,
469
+ act: _Activity, imported: bool = False) -> list[Row]:
470
+ """The human and steer rows of a prompt someone typed. Threads a program drives (subagents,
471
+ `codex exec`, MCP calls, another agent's integration, scheduled automations) are not human
472
+ turns, and get none."""
473
+ after_intr = bool(act.carried.pop("intr", False))
474
+ text, images = prompt
475
+ if not imported:
476
+ Link(act.keys, act.carried.setdefault("link", {})).note_text(text)
477
+ sess = _session(cur, path)
478
+ if sess.run_origin in ("subagent", "automated") or is_scheduled_automation(text):
479
+ return []
480
+ # No tokens; a stable per-session ordinal makes the id deterministic so a
481
+ # re-read (file reset) re-emits the same id and the sink dedups it. The ordinal
482
+ # Includes imported human messages so existing native-tail IDs stay unchanged.
483
+ cur["um_ord"] += 1
484
+ if imported:
485
+ return []
486
+ ts = o.get("timestamp")
487
+ sid = cur["session_id"] or os.path.basename(path)
488
+ return [
489
+ human_event(NAME, (sid, ts or "", cur["um_ord"]), ts, sess, project_name(cur["cwd"])),
490
+ ActivityEvent(
491
+ event_id=make_event_id(NAME, "steer", sid, ts or "", cur["um_ord"]),
492
+ ts=str_or_none(ts), agent=NAME, kind="steer", session_id=sess.session_id,
493
+ turn_id=sess.turn_id, attrs={"via": "typed", "words": len(text.split()),
494
+ "chars": len(text), "images": images,
495
+ "after_interrupt": after_intr}),
496
+ ]
497
+
498
+
499
+ def _chunk_rows(path: str, recs: list[ParsedRecord], cur: FileState, keys: RepoKeys,
500
+ skipped: Counter[str], imports: dict[str, datetime] | None = None) -> list[Row]:
501
+ """All rows of one file's new records; updates `cur`, the private copy of its watermark entry."""
502
+ # shortcut: judged per chunk (plus earlier chunks of the file), so an older-format turn
503
+ # split exactly between its UserMessage item and its user_message counts twice; older
504
+ # files are complete on disk and read in one chunk, upgrade if that changes.
505
+ if any(t == "event_msg" and p.get("type") == "user_message" for _o, t, p, _c in recs):
506
+ cur["has_um"] = True
507
+ act = _Activity(cur, path, keys)
508
+ events: list[Row] = []
509
+ for o, t, p, copied in recs:
510
+ try:
511
+ previous_turn = cur.get("turn")
512
+ _note_context(t, p, copied, cur, keys)
513
+ imported = imported_record(o, (imports or {}).get(_session(cur, path).session_id))
514
+ if imported:
515
+ cur["turn"] = previous_turn
516
+ if t == "event_msg" and p.get("type") == "token_count":
517
+ event = _token_event(o, p, cur, path)
518
+ if event and not imported:
519
+ events.append(event)
520
+ continue
521
+ if copied: # the parent's turns, not this thread's (usage above is unchanged)
522
+ continue
523
+ prompt = prompt_of(t, p, bool(cur.get("import_has_um") if imported
524
+ else cur.get("has_um")))
525
+ if prompt is not None:
526
+ events.extend(_prompt_rows(o, prompt, cur, path, act, imported))
527
+ continue
528
+ if imported:
529
+ continue
530
+ best_effort(act.errors, act.record, o)
531
+ except MALFORMED_RECORD: # one bad line must not wedge every later scan at this offset
532
+ skipped["malformed"] += 1
533
+ skipped.update(act.errors)
534
+ return events + act.events
535
+
536
+
537
+ def _prepare_import(path: str, cur: FileState, off: int, keys: RepoKeys,
538
+ imports: dict[str, datetime]) -> None:
539
+ sid = (cur.get("meta") or {}).get("id")
540
+ if not isinstance(sid, str):
541
+ head = first_record(path)
542
+ payload = head.get("payload") if head else None
543
+ sid = payload.get("id") or payload.get("session_id") if isinstance(payload, dict) else None
544
+ if not isinstance(sid, str) or sid not in imports:
545
+ return
546
+ bound = imports[sid]
547
+ if cur.get("import_checked") == bound.isoformat():
548
+ return
549
+ # A later native format must not renumber the completed imported prefix.
550
+ modern = False
551
+ try:
552
+ with open(path, "rb") as stream:
553
+ for line in stream:
554
+ record = parse_record(line)
555
+ if record and imported_record(record, bound) and record.get("type") == "event_msg":
556
+ p = record.get("payload")
557
+ if isinstance(p, dict) and p.get("type") == "user_message":
558
+ modern = True
559
+ break
560
+ if off:
561
+ prior = _fresh_state(Path(path).stat())
562
+ prior["import_has_um"] = modern
563
+ with open(path, "rb") as stream:
564
+ consumed, chunk_start = 0, 0
565
+ lines: list[bytes] = []
566
+ for line in stream:
567
+ if consumed + len(line) > off:
568
+ break
569
+ lines.append(line)
570
+ consumed += len(line)
571
+ if len(lines) == 1000 or consumed == off:
572
+ records = parsed_records(lines, chunk_start, prior, Counter())
573
+ _chunk_rows(path, records, prior, keys, Counter(), imports)
574
+ lines = []
575
+ chunk_start = consumed
576
+ cur["act"] = prior.get("act", {})
577
+ cur["turn"] = prior.get("turn")
578
+ except OSError as exc:
579
+ raise CollectorUnavailable("codex: imported carried state cannot be checked") from exc
580
+ cur["import_has_um"], cur["import_checked"] = modern, bound.isoformat()
581
+
582
+
583
+ def _trim_state(cur: FileState) -> None:
584
+ carried = cur["act"]
585
+ agg = carried.get("agg")
586
+ if agg is not None and not (agg["calls"] or agg["errors"]):
587
+ del carried["agg"]
588
+ if not carried.get("link"):
589
+ carried.pop("link", None)
590
+ if not any(carried.values()):
591
+ cur.pop("act", None)
592
+
593
+
594
+ def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[Batch]:
595
+ # The discovered files decide: a home can hold archived_sessions with no sessions/ at all.
596
+ paths = session_files(cfg.home)
597
+ if not paths:
598
+ return
599
+ keys = RepoKeys(cfg.repo_key_salt)
600
+ imports = imported_until(cfg.home)
601
+ _upgrade_state(wm, keys)
602
+
603
+ def chunk_rows(path: str, cur: FileState, off: int, lines: list[bytes],
604
+ skipped: Counter[str]) -> list[Row]:
605
+ if imports:
606
+ _prepare_import(path, cur, off, keys, imports)
607
+ events = _chunk_rows(path, parsed_records(lines, off, cur, skipped), cur, keys, skipped,
608
+ imports)
609
+ _trim_state(cur)
610
+ return events
611
+
612
+ yield from file_batches(NAME, paths, wm["files"], _fresh_state, chunk_rows)