agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,484 @@
1
+ """Claude Code collector: ~/.claude/projects/**/*.jsonl
2
+
3
+ One row per `assistant` record. Token truth lives in message.usage. IMPORTANT: most of the
4
+ prompt rides in the cache fields (cache_read/creation), not input_tokens - this is exactly
5
+ the field people miss when they report a "174x undercount". We capture all four separately.
6
+ Subscription-safe (client writes these regardless of OAuth vs API-key auth).
7
+
8
+ Robustness: each record is parsed in isolation, so one corrupt line is skipped (and counted)
9
+ rather than wedging every future scan at the same offset. Note: if Anthropic ever RENAMES a
10
+ usage key, an absent key reads as 0 and would undercount silently - the v2 OTLP lane is the
11
+ intended cross-check for that.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import glob
17
+ import os
18
+ import re
19
+ import time
20
+ from collections import Counter
21
+ from datetime import datetime
22
+ from pathlib import Path
23
+ from typing import Any, Iterable, Iterator, NamedTuple, TypedDict
24
+
25
+ from ..config import Config
26
+ from ..diagnostics import observe_client
27
+ from ..event import (Row, ActivityEvent, Batch, CollectorUnavailable, EventSession, RunOrigin,
28
+ UsageEvent, make_event_id)
29
+ from ..repo_keys import RepoKeys
30
+ from ..state import parse_record
31
+ from ..sources import SourceError, engine_homes
32
+ from .activity_common import IDLE_S, best_effort, drop_raw_shas, human_event, pad_pending, text_of
33
+ from ._util import MALFORMED_RECORD, count_field, file_batches, project_name, str_or_none
34
+ from .claude_code_activity import Activity, ActivityState
35
+
36
+ NAME = "claude_code"
37
+ # The watermark's format. Older state is upgraded once by _upgrade_state; delete a step from there
38
+ # once no machine can still hold state that old.
39
+ _STATE_V = 2
40
+
41
+
42
+ class FileState(TypedDict, total=False):
43
+ """One file's watermark entry. An entry saved by an older version also holds an unread `dev`."""
44
+ off: int
45
+ ino: int
46
+ turn: str # the turn the next record belongs to, until its next user record
47
+ old_loop_seen: bool
48
+ act: ActivityState
49
+
50
+
51
+ def projects_dir(home: Path) -> Path:
52
+ return home / ".claude" / "projects"
53
+
54
+
55
+ def projects_dirs(home: Path) -> list[Path]:
56
+ """Native Claude homes explicitly configured by the user or T3."""
57
+ try:
58
+ roots: dict[Path, Path] = {}
59
+ for home_root in engine_homes(home, NAME):
60
+ root = home_root / "projects"
61
+ roots.setdefault(root.resolve(), root)
62
+ return list(roots.values())
63
+ except SourceError as exc:
64
+ raise CollectorUnavailable(str(exc)) from exc
65
+
66
+
67
+ def main_session_files(home: Path) -> list[str]:
68
+ """Main sessions only (<project>/<id>.jsonl); subagents sit deeper and number tens of thousands."""
69
+ return [path for root in projects_dirs(home)
70
+ for path in glob.glob(str(root / "*" / "*.jsonl"))]
71
+
72
+
73
+ def project_session_files(home: Path, project: str) -> list[str]:
74
+ """The main sessions of one working directory; Claude Code names the directory after its path."""
75
+ name = re.sub(r"[^A-Za-z0-9]", "-", project)
76
+ return [str(f) for root in projects_dirs(home) for f in (root / name).glob("*.jsonl")]
77
+
78
+
79
+ def is_interactive(path: str | Path) -> bool:
80
+ """A main session started at a terminal (entrypoint cli), from its first records."""
81
+ try:
82
+ with open(path, encoding="utf-8", errors="replace") as fh:
83
+ for _, line in zip(range(200), fh):
84
+ if '"entrypoint"' in line:
85
+ o = parse_record(line)
86
+ return o is not None and o.get("entrypoint") == "cli"
87
+ except OSError: # unreadable: it cannot be shown to be interactive
88
+ return False
89
+ return False
90
+
91
+
92
+ def _is_human_text(content: object) -> bool:
93
+ """True for a genuine typed prompt: a plain string (typed text or a slash command) or a
94
+ non-empty block list with no tool_result (text, possibly with pasted images). False for the
95
+ tool_result block-lists Claude Code writes as role=user after each tool call, so those
96
+ injections don't count as you prompting."""
97
+ if isinstance(content, str):
98
+ return True
99
+ if isinstance(content, list):
100
+ return bool(content) and all(
101
+ isinstance(b, dict) and b.get("type") != "tool_result" for b in content)
102
+ return False
103
+
104
+
105
+ # entrypoint values that mean an SDK program is driving the session, not a person at a terminal.
106
+ # Their prompts are ordinary role=user text and pass every other check, so entrypoint is the only
107
+ # thing separating "you asked for a security review" from "a harness asked for one 700 times".
108
+ _PROGRAM_ENTRYPOINTS = {"sdk-py", "sdk-cli"}
109
+
110
+ # Newer Claude Code stamps who started each turn. turnOrigin is authoritative when present: any
111
+ # value but "human" (task_notification, peer, scheduled, system, sdk, auto_continuation) is
112
+ # positively not a person typing. promptSource backs it up on records that carry only that field.
113
+ _NON_HUMAN_PROMPT_SOURCES = {"system", "sdk"}
114
+
115
+
116
+ def _origin_not_human(o: dict[str, Any]) -> bool:
117
+ origin = o.get("turnOrigin")
118
+ if isinstance(origin, str) and origin != "human":
119
+ return True
120
+ return o.get("promptSource") in _NON_HUMAN_PROMPT_SOURCES
121
+
122
+
123
+ # Machine-generated text Claude Code injects AS a role=user turn. These are not tool_results, so
124
+ # _is_human_text accepts them, but nobody typed them. Still needed after turnOrigin: older files
125
+ # lack it, and even current ones write some peer messages without it.
126
+ _INJECTED_PREFIXES = (
127
+ "<task-notification>", # a background task reporting completion back into the session
128
+ "<local-command-stdout>", # output of a slash command echoed back ("Goodbye!")
129
+ "[Request interrupted", # the harness noting you pressed escape -- an action, not a prompt
130
+ "Another Claude session sent a message", # a peer session messaging this one
131
+ "[Cross-session idle notice]", # the harness reporting another session went idle
132
+ "This session is being continued from a previous", # compaction summary without its flag
133
+ "<bash-stdout>", # output of a `!` shell command (the typed `<bash-input>` stays)
134
+ "<bash-stderr>",
135
+ "Stop hook feedback:", # a Stop hook's verdict fed back as a user turn (AgentsView, MIT)
136
+ )
137
+ # Context blocks the harness or IDE attaches to a turn. A turn made of nothing else was not typed
138
+ # (filter ported from AgentsView, github.com/kenn-io/agentsview, MIT licence).
139
+ CONTEXT_BLOCK_RE = re.compile(
140
+ r"<(system-reminder|ide_selection|ide_opened_file)>.*?</\1>", re.S)
141
+ _USAGE_CMD = "<command-name>/usage</command-name>"
142
+
143
+
144
+ _LOOP_RE = re.compile(r"<command-(?:message>(?:[\w-]+:)?loop</command-message|name>/loop</command-name)>")
145
+
146
+
147
+ def is_old_loop(o: dict[str, Any], text: str) -> bool:
148
+ """A /loop record from before Claude Code wrote turnOrigin. Then, every scheduled firing
149
+ looked exactly like the /loop you typed; newer files mark all repeats `scheduled`."""
150
+ return "turnOrigin" not in o and bool(_LOOP_RE.search(text))
151
+
152
+
153
+ def _is_injected(text: str) -> bool:
154
+ """True for harness-generated role=user turns. A `<command-name>` slash-command expansion is
155
+ deliberately NOT here: you typed the slash command, so it stays a human turn."""
156
+ return (text.lstrip().startswith(_INJECTED_PREFIXES)
157
+ or (bool(text.strip()) and not CONTEXT_BLOCK_RE.sub("", text).strip()))
158
+
159
+
160
+ def human_text(o: dict[str, Any]) -> str | None:
161
+ """The text of a record a person typed, else None. A real human prompt is string content
162
+ (typed text or a slash command) or a non-empty block list with no tool_result. Skipped: the tool_result
163
+ user-lines Claude Code injects after every tool call (they dominate the raw "user" count),
164
+ synthetic meta/compact/sidechain lines, and an SDK program driving the session (unknown or
165
+ absent entrypoints stay counted: only what is positively a program is dropped)."""
166
+ if o.get("type") != "user" or o.get("isMeta") or o.get("isCompactSummary") \
167
+ or o.get("isSidechain"):
168
+ return None
169
+ if o.get("entrypoint") in _PROGRAM_ENTRYPOINTS or o.get("entrypoint") == "t3-code" \
170
+ or _origin_not_human(o):
171
+ return None
172
+ m = o.get("message")
173
+ if not isinstance(m, dict) or not _is_human_text(m.get("content")):
174
+ return None
175
+ text = text_of(m.get("content"))
176
+ return None if _is_injected(text) else text
177
+
178
+
179
+ class FileIdentity(NamedTuple):
180
+ subagent: bool
181
+ session_id: str # the fallback when a record does not say
182
+ parent: str | None # the same
183
+
184
+
185
+ def _file_identity(base: Path, path: str) -> FileIdentity:
186
+ """What the path alone says. A subagent lives at
187
+ <parent>/subagents/[workflows/<wf>/]agent-<id>.jsonl; a main session at <sessionId>.jsonl."""
188
+ parts = Path(path).relative_to(base).parts
189
+ stem = Path(path).stem
190
+ if "subagents" in parts[1:-1]: # parts[0] is the project dir, whatever its name
191
+ i = parts.index("subagents", 1)
192
+ return FileIdentity(True, stem.removeprefix("agent-"), parts[i - 1])
193
+ return FileIdentity(False, stem, None)
194
+
195
+
196
+ def _session_of(o: dict[str, Any], ident: FileIdentity, turn: str | None) -> EventSession:
197
+ observe_client(o.get("entrypoint"))
198
+ sub, sid, parent = ident
199
+ if sub: # records carry sessionId = the PARENT session and agentId = their own
200
+ return EventSession(str_or_none(o.get("agentId")) or sid,
201
+ str_or_none(o.get("sessionId")) or parent,
202
+ "subagent", turn)
203
+ ep = o.get("entrypoint")
204
+ origin: RunOrigin = ("automated" if ep in _PROGRAM_ENTRYPOINTS
205
+ else "interactive" if ep == "cli" else "unknown")
206
+ return EventSession(str_or_none(o.get("sessionId")) or sid, None, origin, turn)
207
+
208
+
209
+ def _is_fully_read(path: str, off: int) -> bool:
210
+ """Read to its end and untouched for IDLE_S: no more records will follow."""
211
+ try:
212
+ st = os.stat(path)
213
+ except OSError:
214
+ return False
215
+ return off >= st.st_size and time.time() - st.st_mtime >= IDLE_S
216
+
217
+
218
+ def _upgrade_state(wm: dict[str, Any]) -> None:
219
+ """Bring state saved by an older version to the current shape, once: pending commit calls with
220
+ their target, and link SHAs keyed. Each step is safe on state already upgraded."""
221
+ if wm.get("state_v") == _STATE_V:
222
+ return
223
+ for entry in wm.setdefault("files", {}).values():
224
+ pad_pending(entry.get("act"))
225
+ drop_raw_shas(entry.get("act"))
226
+ wm["state_v"] = _STATE_V
227
+
228
+
229
+ def _record_uid(o: dict[str, Any], path: str) -> str:
230
+ return o.get("uuid") or f"{path}:{o.get('timestamp')}"
231
+
232
+
233
+ def steer_texts(home: Path, sessions: set[str], want: set[str], *,
234
+ paths: Iterable[str] | None = None) -> dict[str, str]:
235
+ """steer event_id -> its raw text, for the ids in `want`, from the main logs of `sessions`. The
236
+ id comes from the same record uid the steer row was made with."""
237
+ out: dict[str, str] = {}
238
+ for f in main_session_files(home) if paths is None else paths:
239
+ if Path(f).stem not in sessions:
240
+ continue
241
+ with open(f, encoding="utf-8", errors="replace") as fh:
242
+ for line in fh:
243
+ if '"user"' not in line:
244
+ continue
245
+ o = parse_record(line) # a malformed line holds no steer to show
246
+ if o is None or o.get("type") != "user":
247
+ continue
248
+ eid = make_event_id(NAME, "steer", _record_uid(o, f))
249
+ m = o.get("message")
250
+ if eid in want and isinstance(m, dict):
251
+ out[eid] = text_of(m.get("content"))
252
+ return out
253
+
254
+
255
+ def agent_replies(home: Path, sessions: set[str], want: set[str], until: datetime | None = None,
256
+ *, paths: Iterable[str] | None = None,
257
+ ) -> dict[str, tuple[str, str]]:
258
+ """steer event_id -> (the agent's last text before it, the agent's last text after it and
259
+ before the next steer in `want`), from the main logs of `sessions`. Text blocks only: no tool
260
+ output and no thinking. A record at or after `until` is not read, so a replay sees only what
261
+ was said by then."""
262
+ out: dict[str, tuple[str, str]] = {}
263
+ for f in main_session_files(home) if paths is None else paths:
264
+ if Path(f).stem not in sessions:
265
+ continue
266
+ last, cur = "", None
267
+ with open(f, encoding="utf-8", errors="replace") as fh:
268
+ for line in fh:
269
+ o = parse_record(line) # a malformed line holds no reply to show
270
+ if o is None:
271
+ continue
272
+ ts = _record_ts(o)
273
+ if until and ts and ts >= until:
274
+ break
275
+ m = o.get("message")
276
+ if o.get("type") == "assistant" and isinstance(m, dict):
277
+ text = text_of(m.get("content"))
278
+ if text.strip():
279
+ last = text
280
+ if cur:
281
+ out[cur] = (out[cur][0], text)
282
+ elif o.get("type") == "user":
283
+ eid = make_event_id(NAME, "steer", _record_uid(o, f))
284
+ if eid in want:
285
+ out[eid], cur = (last, ""), eid
286
+ return out
287
+
288
+
289
+ def _record_ts(o: dict[str, Any]) -> datetime | None:
290
+ try:
291
+ return datetime.fromisoformat(str(o.get("timestamp")).replace("Z", "+00:00"))
292
+ except ValueError:
293
+ return None
294
+
295
+
296
+ def _usage_event(o: dict[str, Any], path: str, ident: FileIdentity,
297
+ turn: str | None) -> tuple[str, UsageEvent] | None:
298
+ """(dedup key, usage row) of an assistant record, None if it has no tokens. Raises on malformed
299
+ token counts."""
300
+ msg = o.get("message")
301
+ if not isinstance(msg, dict):
302
+ return None
303
+ u = msg.get("usage")
304
+ if not isinstance(u, dict):
305
+ return None
306
+ inp = count_field(u, "input_tokens")
307
+ out = count_field(u, "output_tokens")
308
+ cr = count_field(u, "cache_read_input_tokens")
309
+ cw = count_field(u, "cache_creation_input_tokens")
310
+ total = inp + out + cr + cw
311
+ if total == 0:
312
+ return None
313
+ msg_id = msg.get("id")
314
+ # message.id groups a message's blocks; requestId disambiguates a retried message.
315
+ # Fall back to uuid / path:ts only for old transcripts that lack message.id.
316
+ key = f"{msg_id}|{o.get('requestId')}" if msg_id else _record_uid(o, path)
317
+ return key, UsageEvent(
318
+ event_id=make_event_id(NAME, key),
319
+ ts=o.get("timestamp"),
320
+ agent=NAME,
321
+ model=str_or_none(msg.get("model")),
322
+ provider="anthropic",
323
+ input_tokens=inp,
324
+ output_tokens=out,
325
+ cache_read_tokens=cr,
326
+ cache_write_tokens=cw,
327
+ total_tokens=total,
328
+ project=project_name(o.get("cwd")),
329
+ session=_session_of(o, ident, turn),
330
+ )
331
+
332
+
333
+ def _next_turn(o: dict[str, Any], is_subagent: bool, turn: str | None) -> str | None:
334
+ """turn_id after a user record: the latest promptId (tool_results carry it too); a subagent
335
+ keeps its first one, the parent turn that spawned it."""
336
+ pid = str_or_none(o.get("promptId"))
337
+ return pid if pid and not (is_subagent and turn) else turn
338
+
339
+
340
+ def _keep_fuller(best: dict[str, UsageEvent], o: dict[str, Any], path: str,
341
+ ident: FileIdentity, turn: str | None, skipped: Counter[str]) -> None:
342
+ """Fold an assistant record into best: the fuller record wins, an equal one keeps the first.
343
+
344
+ Claude Code writes ONE JSONL line per content block of an assistant message (text + each
345
+ tool_use), and every one of those lines repeats the SAME cumulative message.usage. Keying by
346
+ uuid (unique per line) would count a 6-block message 6x. So dedup by (message.id, requestId)
347
+ within the scan, keeping the max-total record (also correct if the repeats are
348
+ cumulative-growing rather than identical), and derive event_id from that key so the sink dedups
349
+ the same message across scans too.
350
+
351
+ ts is whichever line last raised the max within THIS chunk, and that is not stable across
352
+ scans: Claude Code re-logs a message more than once, observed both a thousand-plus lines later
353
+ in the same file and with a message's own lines minutes apart. A scan seeing only part of that
354
+ spread emits the same event_id with a different ts, and the sink's key is (event_id, ts) -- a
355
+ TimescaleDB hypertable must carry its partitioning column in the unique index -- so both rows
356
+ persist and the partial one would double-count.
357
+
358
+ Not fixable here: which lines a scan sees is exactly what varies, so no per-chunk rule is
359
+ invariant, and closing it would need a per-message ts cache in the watermark for every tracked
360
+ file. The sink's agents.usage_event_dedup view settles it instead, keeping the largest total
361
+ per event_id (counts are cumulative, so the largest is the complete one)."""
362
+ try:
363
+ found = _usage_event(o, path, ident, turn)
364
+ except MALFORMED_RECORD:
365
+ skipped["malformed"] += 1
366
+ return
367
+ if found and (found[0] not in best or best[found[0]].total_tokens < found[1].total_tokens):
368
+ best[found[0]] = found[1]
369
+
370
+
371
+ def _only_ran_usage(fresh: bool, best: dict[str, UsageEvent],
372
+ steers: list[tuple[ActivityEvent | None, bool]]) -> bool:
373
+ """A session that only ran /usage (no model call, every prompt a /usage) was nobody working
374
+ (AgentsView, MIT). Judged only on a read from byte 0, so a /usage typed mid-session and caught
375
+ alone by a live scan is kept, as a full re-read keeps it. shortcut: a file whose first chunk
376
+ held only the /usage keeps it dropped if real work follows; rare."""
377
+ return fresh and not best and bool(steers) and all(is_usage for _s, is_usage in steers)
378
+
379
+
380
+ def _human_row(o: dict[str, Any], text: str, path: str, sess: EventSession,
381
+ act: Activity) -> tuple[UsageEvent, tuple[ActivityEvent | None, bool]]:
382
+ """The human row of a typed prompt, and its steer row with whether the prompt ran /usage."""
383
+ uid = _record_uid(o, path)
384
+ row = human_event(NAME, (uid,), o.get("timestamp"), sess, project_name(o.get("cwd")))
385
+ return row, (act.steer(o, uid, text, sess), _USAGE_CMD in text)
386
+
387
+
388
+ def _old_loop_before(path: str, off: int) -> bool:
389
+ """Recover the missing flag from an older watermark without re-emitting historical prompts."""
390
+ if not off:
391
+ return False
392
+ with open(path, "rb") as stream:
393
+ while stream.tell() < off:
394
+ raw = stream.readline(off - stream.tell())
395
+ if not raw:
396
+ break
397
+ o = parse_record(raw)
398
+ text = human_text(o) if o else None
399
+ if o and text is not None and is_old_loop(o, text):
400
+ return True
401
+ return False
402
+
403
+
404
+ def _chunk_rows(path: str, base: Path, lines: list[bytes], from_start: bool, cur: FileState,
405
+ keys: RepoKeys, skipped: Counter[str]) -> list[Row]:
406
+ """All rows of one file's new lines, read from byte 0 or not; updates `cur`, the private copy
407
+ of its watermark entry."""
408
+ best: dict[str, UsageEvent] = {}
409
+ human: list[UsageEvent] = []
410
+ ident = _file_identity(base, path)
411
+ # turn_id is carried across chunks in the file's watermark entry, so a turn that began in an
412
+ # earlier scan is not lost.
413
+ # shortcut: files first read before this existed start at NULL until their next user record.
414
+ turn = str_or_none(cur.get("turn"))
415
+ act = Activity(NAME, path, ident.subagent, cur.get("act"), keys)
416
+ steers: list[tuple[ActivityEvent | None, bool]] = []
417
+ for raw in lines:
418
+ o = parse_record(raw)
419
+ if o is None:
420
+ skipped["malformed"] += 1
421
+ continue
422
+ try:
423
+ typ = o.get("type")
424
+ if typ == "user":
425
+ turn = _next_turn(o, ident.subagent, turn)
426
+ sess = _session_of(o, ident, turn)
427
+ best_effort(act.errors, act.record, o, sess)
428
+ if typ == "user":
429
+ text = human_text(o)
430
+ if text is None:
431
+ continue
432
+ if is_old_loop(o, text):
433
+ if cur.get("old_loop_seen"): # only the first was typed
434
+ continue
435
+ cur["old_loop_seen"] = True
436
+ row, steer = _human_row(o, text, path, sess, act)
437
+ human.append(row)
438
+ steers.append(steer)
439
+ elif typ == "assistant":
440
+ _keep_fuller(best, o, path, ident, turn, skipped)
441
+ except MALFORMED_RECORD: # one bad line must not wedge every later scan at this offset
442
+ skipped["malformed"] += 1
443
+ if _is_fully_read(path, cur["off"]):
444
+ act.finish()
445
+ if _only_ran_usage(from_start, best, steers):
446
+ human, steers = [], []
447
+ skipped.update(act.errors)
448
+ if turn:
449
+ cur["turn"] = turn
450
+ else:
451
+ cur.pop("turn", None)
452
+ state = act.state()
453
+ if state:
454
+ cur["act"] = state
455
+ else:
456
+ cur.pop("act", None)
457
+ return [*best.values(), *human, *(s for s, _u in steers if s), *act.events]
458
+
459
+
460
+ def _fresh_state(st: os.stat_result) -> FileState:
461
+ return {"off": 0, "ino": st.st_ino}
462
+
463
+
464
+ def _has_open_link_turn(entry: FileState) -> bool:
465
+ """Such a file is looked at again with no new bytes: once it has gone idle, its turn closes."""
466
+ return bool((entry.get("act") or {}).get("lo"))
467
+
468
+
469
+ def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[Batch]:
470
+ roots = projects_dirs(cfg.home)
471
+ if not any(base.exists() for base in roots):
472
+ return
473
+ _upgrade_state(wm)
474
+ keys = RepoKeys(cfg.repo_key_salt) # cwd -> repo_key cached once per collect() pass
475
+
476
+ def chunk_rows(path: str, cur: FileState, off: int, lines: list[bytes],
477
+ skipped: Counter[str]) -> list[Row]:
478
+ if "old_loop_seen" not in cur:
479
+ cur["old_loop_seen"] = _old_loop_before(path, off)
480
+ return _chunk_rows(path, base, lines, not off, cur, keys, skipped)
481
+
482
+ for base in roots:
483
+ yield from file_batches(NAME, glob.glob(str(base / "**" / "*.jsonl"), recursive=True),
484
+ wm["files"], _fresh_state, chunk_rows, _has_open_link_turn)