agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,592 @@
1
+ """Cursor CLI collector: the `cursor-agent` CLI's chat stores and the Cursor SDK's run stores.
2
+
3
+ CLI chats: ~/.cursor/chats/<h>/<uuid>/store.db (sqlite, opened read-only, nothing copied). Meta
4
+ key '0' is hex-encoded JSON (agentId, createdAt, latestRootBlobId, and subagentInfo on a chat
5
+ another agent spawned). `blobs` holds the messages (JSON) and root blobs (protobuf) that list the
6
+ messages in order as field 1, each a 32-byte blob id. Every step writes a new root. A summarized
7
+ or rewound chat starts a new list, so its older messages sit only in older roots: every root list
8
+ that is not a prefix of another one is read.
9
+
10
+ This collector emits NO token usage (the stores have none, and cursor_api counts these tokens).
11
+ It emits human rows (zero tokens) for the prompts, user messages carrying
12
+ `providerOptions.cursor.requestId` (the others are injected context or a summary), outside
13
+ subagent chats, plus their session rows and activity rows: steer, commit, rejection, turn.
14
+ SDK stores: ~/.cursor/projects/*/sdk-agent-store/*/index.db, one `turn` row per ended run.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import copy
20
+ import glob
21
+ import hashlib
22
+ import json
23
+ import os
24
+ import re
25
+ import sqlite3
26
+ import stat
27
+ import sys
28
+ import time
29
+ from collections import Counter
30
+ from dataclasses import asdict, dataclass, field
31
+ from datetime import datetime, timedelta, timezone
32
+ from pathlib import Path
33
+ from typing import Any, Generator, Iterable, Iterator, NotRequired, TypedDict
34
+
35
+ from ..config import Config
36
+ from ..cursor_run import automated_sessions
37
+ from ..event import (ActivityKind, Batch, ActivityEvent, CollectorUnavailable, EventSession, Row,
38
+ RunOrigin, make_event_id)
39
+ from ..repo_keys import RepoKeys
40
+ from .activity_common import (IDLE_S, Link, TurnAgg, add_call, best_effort, blank_tool_tally, human_event,
41
+ ms_between, record_commits, text_of, turn_attrs)
42
+ from .commits import commit_call, commit_dir, commit_shas
43
+ from ._util import describe_files, epoch_ms_to_iso, str_or_none, warn_skipped
44
+ from .cursor import connect_ro
45
+
46
+ NAME = "cursor_cli"
47
+ _ENDED = {"FINISHED": "finished", "CANCELLED": "cancelled", "ERROR": "error"}
48
+ _SQL_IN_CHUNK = 500
49
+ _TS = re.compile(r"<timestamp>\s*\w+,\s*([A-Za-z]{3})\w*\.?\s+(\d{1,2}),\s*(\d{4}),\s*(\d{1,2}):(\d\d)"
50
+ r"\s*([AP]M)\s*\(UTC(?:([+-])(\d{1,2})(?::(\d\d))?)?\)\s*</timestamp>")
51
+ USER_QUERY = re.compile(r"<user_query>(.*?)</user_query>", re.S)
52
+ EXIT_LINE = re.compile(r"^Exit code: (-?\d+)\s*(?:Command output:\s*)?")
53
+ _IDENT = re.compile(r"[A-Za-z][A-Za-z_]*")
54
+ _UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
55
+
56
+
57
+ def chat_stores(home: Path) -> list[str]:
58
+ """Every chat's store.db; a chat's directory name is its agentId in every store seen."""
59
+ return sorted(glob.glob(str(home / ".cursor" / "chats" / "*" / "*" / "store.db")))
60
+
61
+
62
+ def _cursor_options(m: dict[str, Any]) -> dict[str, Any]:
63
+ """The cursor block of a message's providerOptions."""
64
+ po = m.get("providerOptions")
65
+ c = po.get("cursor") if isinstance(po, dict) else None
66
+ return c if isinstance(c, dict) else {}
67
+
68
+
69
+ def request_id(m: dict[str, Any]) -> str | None:
70
+ """requestId of a real prompt; None for any other message."""
71
+ c = _cursor_options(m)
72
+ r = c.get("requestId")
73
+ return r if m.get("role") == "user" and isinstance(r, str) and r and not c.get("isSummary") \
74
+ else None
75
+
76
+
77
+ def _by_tool_key(name: str) -> str:
78
+ """by_tool key. An MCP tool is `mcp_<server>_<tool>` and a server label is free text, so it
79
+ counts as `mcp`; a name that is not a plain identifier counts as `other`."""
80
+ if name.startswith("mcp") or name == "CallMcpTool":
81
+ return "mcp"
82
+ return name if _IDENT.fullmatch(name) else "other"
83
+
84
+
85
+ def prompt_ts(text: str, created: str | None) -> str | None:
86
+ """The prompt's embedded `<timestamp>Weekday, Mon D, YYYY, H:MM AM (UTC+N)</timestamp>`
87
+ (minute precision), else the chat's createdAt."""
88
+ m = _TS.search(text)
89
+ if not m:
90
+ return created
91
+ mon, day, year, hh, mm, ap, sign, oh, om = m.groups()
92
+ try:
93
+ d = datetime.strptime(f"{mon} {day} {year} {hh}:{mm} {ap}", "%b %d %Y %I:%M %p")
94
+ off = timedelta(hours=int(oh or 0), minutes=int(om or 0)) * (-1 if sign == "-" else 1)
95
+ return d.replace(tzinfo=timezone(off)).isoformat()
96
+ except ValueError:
97
+ return created
98
+
99
+
100
+ def _varint(b: bytes, i: int) -> tuple[int, int]:
101
+ r = s = 0
102
+ while True:
103
+ c = b[i] # IndexError on a truncated blob
104
+ i += 1
105
+ r |= (c & 0x7F) << s
106
+ s += 7
107
+ if not c & 0x80:
108
+ return r, i
109
+
110
+
111
+ def _field1_ids(b: bytes) -> tuple[str, ...]:
112
+ """The 32-byte field-1 entries of a top-level protobuf message, in order."""
113
+ out, i = [], 0
114
+ while i < len(b):
115
+ tag, i = _varint(b, i)
116
+ if tag >> 3 == 0:
117
+ raise ValueError("invalid protobuf field")
118
+ wt = tag & 7
119
+ if wt == 0:
120
+ _, i = _varint(b, i)
121
+ elif wt == 1:
122
+ i += 8
123
+ elif wt == 5:
124
+ i += 4
125
+ elif wt == 2:
126
+ n, i = _varint(b, i)
127
+ if tag >> 3 == 1 and n == 32:
128
+ out.append(b[i:i + n].hex())
129
+ i += n
130
+ else:
131
+ raise ValueError(wt)
132
+ if i > len(b):
133
+ raise ValueError("truncated protobuf field")
134
+ return tuple(out)
135
+
136
+
137
+ def root_lists(con: sqlite3.Connection) -> list[tuple[str, ...]]:
138
+ """Every root list that is no other list's prefix. A root starts with its first field-1
139
+ entry (measured on every root of real stores)."""
140
+ seqs = set()
141
+ for (data,) in con.execute("SELECT data FROM blobs WHERE substr(data, 1, 2) = X'0A20'"):
142
+ try:
143
+ ids = _field1_ids(bytes(data))
144
+ except (ValueError, IndexError): # a message blob that merely opens like a root
145
+ continue
146
+ if ids:
147
+ seqs.add(ids)
148
+ srt = sorted(seqs)
149
+ # in sorted order a list that prefixes another one prefixes the very next one
150
+ return [a for a, b in zip(srt, srt[1:] + [()]) if b[:len(a)] != a]
151
+
152
+
153
+ def messages(con: sqlite3.Connection, ids: set[str],
154
+ skipped: Counter[str] | None = None) -> dict[str, dict[str, Any]]:
155
+ """The listed messages, plus every prompt (a prompt can be in no surviving root)."""
156
+ out: dict[str, dict[str, Any]] = {}
157
+ found: set[str] = set()
158
+
159
+ def take(rows: Iterable[tuple[str, Any]]) -> None:
160
+ for k, v in rows:
161
+ required = k in ids and k not in found
162
+ found.add(k)
163
+ try:
164
+ m = json.loads(v)
165
+ except (ValueError, TypeError): # a listed id can be a protobuf root, not a message
166
+ if skipped is not None and required:
167
+ if v is None:
168
+ skipped["missing_payloads"] += 1
169
+ else:
170
+ try:
171
+ if not isinstance(v, bytes):
172
+ raise ValueError("message is neither JSON nor protobuf")
173
+ _field1_ids(v)
174
+ except (ValueError, IndexError):
175
+ skipped["malformed"] += 1
176
+ continue
177
+ if isinstance(m, dict):
178
+ out[k] = m
179
+ elif skipped is not None and required:
180
+ skipped["malformed"] += 1
181
+ q = sorted(ids)
182
+ for j in range(0, len(q), _SQL_IN_CHUNK):
183
+ part = q[j:j + _SQL_IN_CHUNK]
184
+ marks = ",".join("?" * len(part))
185
+ sql = f"SELECT id, data FROM blobs WHERE id IN ({marks})" # nosec B608 # marks is only ?s
186
+ take(con.execute(sql, part))
187
+ take(con.execute("SELECT id, data FROM blobs WHERE substr(data, 1, 1) = X'7B' "
188
+ "AND instr(data, '\"requestId\"') > 0"))
189
+ if skipped is not None and (missing := len(ids - found)):
190
+ skipped["missing_payloads"] += missing
191
+ return out
192
+
193
+
194
+ class _TurnOrigin(TypedDict):
195
+ """The origin fields a chat's turn rows carry."""
196
+ run_origin: RunOrigin
197
+ parent_session_id: NotRequired[str | None]
198
+
199
+
200
+ @dataclass
201
+ class _Turn:
202
+ """One requestId's turn. A prompt no surviving root lists has no tool messages to read and
203
+ emits no turn or link row (`listed` False)."""
204
+ msg: dict[str, Any]
205
+ prompt_id: str | None = None
206
+ closed: bool = False
207
+ listed: bool = True
208
+ agg: TurnAgg = field(default_factory=blank_tool_tally)
209
+ acts: list[tuple[ActivityKind, str, dict[str, Any]]] = field(default_factory=list) # (kind, seed, attrs) rows, emitted at close
210
+ cids: set[object] = field(default_factory=set) # tool calls counted, by toolCallId
211
+ more: dict[str, Any] = field(default_factory=dict) # follow-up prompt text -> (blob id, message)
212
+ link: dict[str, Any] = field(default_factory=dict)
213
+ sources: set[str] = field(default_factory=set)
214
+
215
+
216
+ class _Activity:
217
+ """One chat's activity extractor. Publish rows with each complete turn snapshot."""
218
+
219
+ def __init__(self, keys: RepoKeys) -> None:
220
+ self.keys = keys
221
+ self.calls: dict[object, Any] = {} # toolCallId -> args, across the chat's lists
222
+ self.errors: Counter[str] = Counter() # message read and extraction gaps
223
+
224
+ def record(self, message: dict[str, Any], turn: _Turn) -> None:
225
+ """One assistant or tool message: tool aggregates, commits, rejections, link."""
226
+ link = Link(self.keys, turn.link)
227
+ content = [b for b in message["content"] if isinstance(b, dict)] \
228
+ if isinstance(message.get("content"), list) else []
229
+ if message.get("role") == "assistant":
230
+ self._assistant(content, turn, link)
231
+ elif message.get("role") == "tool":
232
+ self._tool_message(message, content, turn, link)
233
+
234
+ def _assistant(self, content: list[dict[str, Any]], turn: _Turn, link: Link) -> None:
235
+ link.note_text(text_of(content))
236
+ for b in content:
237
+ if b.get("type") == "tool-call" and isinstance(b.get("toolName"), str):
238
+ cid = b.get("toolCallId")
239
+ self.calls[cid] = b.get("args")
240
+ if isinstance(cid, str) and cid in turn.cids:
241
+ continue # a rewritten blob that a later root lists again
242
+ turn.cids.add(cid)
243
+ add_call(turn.agg, _by_tool_key(b["toolName"]), b.get("args"))
244
+
245
+ def _tool_message(self, message: dict[str, Any], content: list[dict[str, Any]], turn: _Turn, link: Link) -> None:
246
+ hl = _cursor_options(message).get("highLevelToolCallResult")
247
+ err = isinstance(hl, dict) and hl.get("isError") is True
248
+ turn.agg["errors"] += err
249
+ for b in content:
250
+ if b.get("type") != "tool-result":
251
+ continue
252
+ res, cid = b.get("result"), b.get("toolCallId")
253
+ res = res if isinstance(res, str) else ""
254
+ link.note_text(res)
255
+ if b.get("toolName") == "SwitchMode" \
256
+ and res.startswith("Mode switch was rejected by the user"):
257
+ turn.acts.append(("rejection", str(cid), {}))
258
+ elif b.get("toolName") == "Shell":
259
+ self._shell_result(b, res, cid, err, turn, link)
260
+
261
+ def _shell_result(self, b: dict[str, Any], res: str, cid: object, err: bool, turn: _Turn,
262
+ link: Link) -> None:
263
+ turn.agg["hooks"] += res.startswith("Rejected: Hook blocked")
264
+ args = self.calls.get(cid)
265
+ cmd = args.get("command") if isinstance(args, dict) else None
266
+ wd = args.get("working_directory") if isinstance(args, dict) else None
267
+ link.place(3, self.keys.of_cwd(wd))
268
+ if not isinstance(cmd, str) or not (hit := commit_call(cmd)):
269
+ return
270
+ ex = EXIT_LINE.match(res) # "Exit code: N\n\nCommand output:\n..." when it ran
271
+ output = res[ex.end():] if ex else res
272
+ found = commit_shas(output, err or not ex or ex[1] != "0", hit[1])
273
+ for sha, attrs in record_commits(link, wd, commit_dir(cmd), output, found, amend=hit[0]):
274
+ turn.acts.append(("commit", f"{cid}|{sha}", attrs))
275
+
276
+
277
+ def _read_list(seq: tuple[str, ...], msgs: dict[str, dict[str, Any]], done: set[str], act: _Activity,
278
+ turns: dict[str, _Turn], seen: set[str],
279
+ prompts: dict[str, str] | None = None) -> None:
280
+ """One root list: its prompts open and close turns, its other messages belong to the open one."""
281
+ rid = None
282
+ for mid in seq:
283
+ m = msgs.get(mid)
284
+ if m is None:
285
+ continue
286
+ r = request_id(m)
287
+ if r:
288
+ if rid and rid != r:
289
+ turns[rid].closed = True
290
+ rid = r
291
+ first = (prompts or {}).get(r, mid)
292
+ turn = turns.setdefault(r, _Turn(msgs.get(first, {}), prompt_id=first))
293
+ turn.sources.add(mid)
294
+ if first in msgs:
295
+ turn.sources.add(first)
296
+ # A prompt sent while the turn runs reuses its requestId. A blob with the first
297
+ # prompt's text is a rewrite of it, not a new prompt.
298
+ # Published turns retain their original prompt blob when later root order changes.
299
+ text = text_of(m.get("content"))
300
+ if m is not turn.msg and text != text_of(turn.msg.get("content")):
301
+ turn.more.setdefault(text, (mid, m))
302
+ elif rid and rid not in done and mid not in seen:
303
+ seen.add(mid)
304
+ turns[rid].sources.add(mid)
305
+ best_effort(act.errors, act.record, m, turns[rid])
306
+
307
+
308
+ def _read_turns(lists: list[tuple[str, ...]], msgs: dict[str, dict[str, Any]], done: set[str],
309
+ act: _Activity, prompts: dict[str, str] | None = None) -> dict[str, _Turn]:
310
+ """Every requestId's turn, preserving any supplied original prompt anchors."""
311
+ turns: dict[str, _Turn] = {}
312
+ seen: set[str] = set()
313
+ for seq in lists:
314
+ _read_list(seq, msgs, done, act, turns, seen, prompts)
315
+ for mid, m in msgs.items(): # a prompt no surviving root lists
316
+ r = request_id(m)
317
+ if r and r not in turns:
318
+ first = (prompts or {}).get(r, mid)
319
+ turns[r] = _Turn(msgs.get(first, {}), prompt_id=first, listed=False,
320
+ sources={mid})
321
+ return turns
322
+
323
+
324
+ def _turn_rows(rid: str, turn: _Turn, sid: str, human: bool, origin: _TurnOrigin, created: str | None,
325
+ keys: RepoKeys, ts: str | None) -> list[Row]:
326
+ """The rows of one publishable turn snapshot."""
327
+ rows: list[Row] = []
328
+
329
+ def emit(kind: ActivityKind, seed: str, attrs: dict[str, Any], at: str | None = ts) -> None:
330
+ rows.append(ActivityEvent(event_id=make_event_id(NAME, kind, f"{sid}|{seed}"), ts=at,
331
+ agent=NAME, kind=kind, session_id=sid, turn_id=rid, attrs=attrs))
332
+ # the first prompt keeps the (sid, rid) ids; a follow-up adds its blob id to them
333
+ prompts = [((), turn.msg)] + [((mid,), pm) for mid, pm in turn.more.values()]
334
+ # A missing original blob cannot establish which surviving prompts were human follow-ups.
335
+ for extra, pm in prompts if human and turn.msg else ():
336
+ ptext = text_of(pm.get("content"))
337
+ pts = prompt_ts(ptext, created)
338
+ q = USER_QUERY.search(ptext)
339
+ words = (q[1] if q else ptext).strip()
340
+ c = pm.get("content")
341
+ images = sum(isinstance(b, dict) and b.get("type") == "image" for b in c) \
342
+ if isinstance(c, list) else 0
343
+ if turn.listed:
344
+ Link(keys, turn.link).note_text(ptext)
345
+ rows.append(human_event(NAME, (sid, rid, *extra), pts,
346
+ EventSession(sid, None, "interactive", rid)))
347
+ emit("steer", "|".join((rid, *extra)), {"via": "typed", "words": len(words.split()),
348
+ "chars": len(words), "images": images}, pts)
349
+ for kind, seed, attrs in turn.acts:
350
+ emit(kind, seed, attrs)
351
+ if turn.listed:
352
+ # The sink replaces this snapshot only when its keyed source coverage contains the old one.
353
+ sources = sorted(k for mid in turn.sources
354
+ if (k := keys.of(f"cursor_cli:source:{sid}:{mid}")))
355
+ revision = {"revision_sources": sources} if sources else {}
356
+ emit("turn", rid, {**turn_attrs(turn.agg), "hook_blocks": turn.agg["hooks"], **origin,
357
+ **revision})
358
+ emit("link", rid, {**Link(keys, turn.link).attrs(origin["run_origin"]), **revision})
359
+ return rows
360
+
361
+
362
+ def _closed_turn_rows(path: str, con: sqlite3.Connection, meta: dict[str, Any], cur: dict[str, Any], idle: bool,
363
+ keys: RepoKeys) -> tuple[list[Row], Counter[str]]:
364
+ """Publish new or changed turn snapshots; update `cur`, a private checkpoint copy."""
365
+ sid = str_or_none(meta.get("agentId")) or os.path.basename(os.path.dirname(path))
366
+ # A subagent chat's prompts are its parent agent talking, not a person. Its parent is in
367
+ # subagentInfo.parentAgentId, but it has no usage or human row to carry a session row.
368
+ info = meta.get("subagentInfo")
369
+ parent = info.get("parentAgentId") if isinstance(info, dict) else None
370
+ created = epoch_ms_to_iso(meta.get("createdAt"))
371
+ act = _Activity(keys)
372
+ if isinstance(meta.get("latestRootBlobId"), str):
373
+ root = con.execute("SELECT data FROM blobs WHERE id = ?",
374
+ (meta["latestRootBlobId"],)).fetchone()
375
+ if root is None or root[0] is None:
376
+ act.errors["missing_payloads"] += 1
377
+ elif not isinstance(root[0], bytes):
378
+ act.errors["malformed"] += 1
379
+ else:
380
+ try:
381
+ _field1_ids(root[0])
382
+ except (ValueError, IndexError):
383
+ act.errors["malformed"] += 1
384
+ lists = root_lists(con)
385
+ msgs = messages(con, {i for seq in lists for i in seq}, act.errors)
386
+ # Read after the message snapshot: a concurrent wrapper registers before its first prompt.
387
+ scripted = sid in automated_sessions()
388
+ human = not info and not scripted
389
+ origin: _TurnOrigin = {"run_origin": "automated"} if scripted else (
390
+ {"run_origin": "interactive"} if human else {
391
+ "run_origin": "subagent",
392
+ "parent_session_id": parent if isinstance(parent, str) and _UUID.fullmatch(parent) else None})
393
+ done = set(cur.get("done", ()))
394
+ snapshots = cur.setdefault("snapshots", {})
395
+ prompts = {rid: snapshot["prompt"] for rid, snapshot in snapshots.items()}
396
+ previous = con.execute("SELECT data FROM blobs WHERE id = ?", (cur.get("root"),)).fetchone()
397
+ if previous and isinstance(previous[0], bytes):
398
+ try:
399
+ ids = _field1_ids(previous[0])
400
+ except (ValueError, IndexError):
401
+ ids = ()
402
+ for mid in ids:
403
+ if r := request_id(msgs.get(mid, {})):
404
+ prompts.setdefault(r, mid)
405
+ text = text_of(msgs[mid].get("content"))
406
+ if text != text_of(msgs.get(prompts[r], {}).get("content")) \
407
+ and (key := keys.of(f"cursor_cli:prompt:{sid}:{r}:{text}")):
408
+ snapshot = snapshots.setdefault(r, {"prompt": prompts[r]})
409
+ snapshot.setdefault("followups", {}).setdefault(key, mid)
410
+ turns = _read_turns(lists, msgs, set(), act, prompts)
411
+ events: list[Row] = []
412
+ for rid, turn in turns.items():
413
+ # Idle publishes a revisable snapshot because Cursor has no turn-end record.
414
+ if not (turn.closed or idle or rid in done):
415
+ continue
416
+ snapshot = snapshots.get(rid, {})
417
+ followups = dict(snapshot.get("followups", {}))
418
+ for text, (mid, pm) in turn.more.items():
419
+ if key := keys.of(f"cursor_cli:prompt:{sid}:{rid}:{text}"):
420
+ turn.more[text] = (followups.setdefault(key, mid), pm)
421
+ # Ordinary activity rows also need their original timestamp if the prompt blob disappears.
422
+ ts = snapshot.get("ts", prompt_ts(text_of(turn.msg.get("content")), created))
423
+ rows = _turn_rows(rid, turn, sid, human, origin, created, keys, ts)
424
+ fingerprint = hashlib.sha256(json.dumps(
425
+ [asdict(row) for row in rows], sort_keys=True).encode()).hexdigest()
426
+ snapshots[rid] = {"prompt": turn.prompt_id, "followups": followups,
427
+ "ts": ts, "fingerprint": fingerprint}
428
+ if snapshot.get("fingerprint") == fingerprint:
429
+ continue
430
+ done.add(rid)
431
+ events += rows
432
+ cur["done"] = sorted(done)
433
+ cur["open"] = sum(r not in done for r in turns)
434
+ return events, act.errors
435
+
436
+
437
+ def chat_meta(con: sqlite3.Connection) -> dict[str, Any] | None:
438
+ row = con.execute("SELECT value FROM meta WHERE key = '0'").fetchone()
439
+ try:
440
+ m = json.loads(bytes.fromhex(row[0]))
441
+ except (TypeError, ValueError):
442
+ return None
443
+ return m if isinstance(m, dict) else None
444
+
445
+
446
+ def _mtime(path: str) -> float:
447
+ return max((os.stat(p).st_mtime for p in (path, path + "-wal") if os.path.exists(p)),
448
+ default=0.0)
449
+
450
+
451
+ def _finished_run_rows(path: str, have: set[str]) -> list[ActivityEvent] | None:
452
+ """One turn row per run that ended and was not sent yet; None if the store cannot be read.
453
+ Queued and running runs wait: a row is insert-only, so one stored as running would never
454
+ become finished."""
455
+ con = connect_ro(path)
456
+ if con is None:
457
+ return None
458
+ try:
459
+ rows = con.execute("SELECT run_id, agent_id, status, created_at, started_at, finished_at "
460
+ "FROM runs").fetchall()
461
+ except sqlite3.DatabaseError:
462
+ return None
463
+ finally:
464
+ con.close()
465
+ out = []
466
+ for run_id, agent_id, status, created, started, finished in rows:
467
+ st = _ENDED.get(status) # error_code and result are free text: never read
468
+ if not st or not isinstance(run_id, str) or run_id in have:
469
+ continue
470
+ out.append(ActivityEvent(
471
+ event_id=make_event_id(NAME, "turn", f"{agent_id}|{run_id}"), ts=started or created,
472
+ agent=NAME, kind="turn", session_id=agent_id, turn_id=run_id,
473
+ attrs={"status": st, "duration_ms": ms_between(started, finished),
474
+ "run_origin": "automated"}))
475
+ return out
476
+
477
+
478
+ def describe(cfg: Config, wm: dict[str, Any]) -> str:
479
+ if "chats" in wm:
480
+ return f"{len(wm['chats'])} chats, {len(wm.get('sdk', {}))} SDK stores tracked"
481
+ return describe_files(cfg, wm)
482
+
483
+
484
+ def _closed_source(path: str) -> list[list[int] | None] | None:
485
+ """Main/WAL identity for a closed chat; journals and unreadable metadata cannot be cached."""
486
+ # Windows ctime is creation time, so restored mtime can hide a write.
487
+ if sys.platform == "win32":
488
+ return None
489
+ files: list[list[int] | None] = []
490
+ try:
491
+ if os.path.lexists(path + "-journal"):
492
+ return None
493
+ for source in (path, path + "-wal"):
494
+ try:
495
+ info = os.stat(source)
496
+ except FileNotFoundError:
497
+ if source == path:
498
+ return None
499
+ files.append(None)
500
+ continue
501
+ if not stat.S_ISREG(info.st_mode):
502
+ return None
503
+ files.append([info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns,
504
+ info.st_ctime_ns, info.st_mode])
505
+ return None if os.path.lexists(path + "-journal") else files
506
+ except OSError:
507
+ return None
508
+
509
+
510
+ def _closed_checkpoint(cur: dict[str, Any]) -> bool:
511
+ root = cur.get("root")
512
+ return (isinstance(root, str) and re.fullmatch(r"[0-9a-f]{64}", root) is not None
513
+ and type(cur.get("open")) is int and cur["open"] == 0
514
+ and isinstance(cur.get("snapshots"), dict))
515
+
516
+
517
+ def _read_chat(path: str, prev: dict[str, Any], keys: RepoKeys) -> tuple[list[Any], dict[str, Any], Counter[str]] | None:
518
+ """(rows, the chat's new watermark entry, messages skipped); None when it has nothing new.
519
+ Raises sqlite3.DatabaseError when a required read finds the store locked or corrupt."""
520
+ source = _closed_source(path)
521
+ # shortcut: cached closed data skips lock checks, require SQLite opens for fresh lock health.
522
+ if source is not None and _closed_checkpoint(prev) and prev.get("closed_source") == source:
523
+ return None
524
+ con = connect_ro(path)
525
+ if con is None:
526
+ raise sqlite3.OperationalError("store could not be opened")
527
+ try:
528
+ # A deep copy: only advance_watermark() may publish this chat's state into the watermark.
529
+ cur = copy.deepcopy(prev)
530
+ cur.pop("closed_source", None)
531
+ meta = chat_meta(con)
532
+ if meta is None:
533
+ return ([], cur, Counter()) if cur != prev else None
534
+ events, errs = _closed_turn_rows(path, con, meta, cur, time.time() - _mtime(path) >= IDLE_S, keys)
535
+ cur["root"] = meta.get("latestRootBlobId")
536
+ if not errs and _closed_checkpoint(cur) and source is not None and source == _closed_source(path):
537
+ cur["closed_source"] = source
538
+ return events, cur, errs
539
+ finally:
540
+ con.close()
541
+
542
+
543
+ def _sdk_batches(home: Path, sdk: dict[str, Any]) -> Generator[Batch, None, int]:
544
+ """The SDK stores' batches; returns how many stores could not be read."""
545
+ unreadable = 0
546
+ for path in sorted(glob.glob(str(home / ".cursor" / "projects" / "*" / "sdk-agent-store"
547
+ / "*" / "index.db"))):
548
+ have = set(sdk.get(path, {}).get("runs", ()))
549
+ runs = _finished_run_rows(path, have)
550
+ if runs is None:
551
+ unreadable += 1
552
+ continue
553
+ if not runs:
554
+ continue
555
+
556
+ def advance_sdk_watermark(p: str = path, seen: list[str] = sorted(
557
+ have | {str(e.turn_id) for e in runs})) -> None:
558
+ sdk[p] = {"runs": seen}
559
+
560
+ yield runs, advance_sdk_watermark
561
+ return unreadable
562
+
563
+
564
+ def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[Batch]:
565
+ chats: dict[str, Any] = wm.setdefault("chats", {})
566
+ sdk: dict[str, Any] = wm.setdefault("sdk", {})
567
+ keys = RepoKeys(cfg.repo_key_salt)
568
+ skipped: Counter[str] = Counter()
569
+ unreadable = 0
570
+ for path in chat_stores(cfg.home):
571
+ prev = chats.get(path) or {}
572
+ try:
573
+ read = _read_chat(path, prev, keys)
574
+ except sqlite3.DatabaseError: # locked or corrupt: counted below, the next scan retries
575
+ unreadable += 1
576
+ continue
577
+ if read is None:
578
+ continue
579
+ events, cur, errs = read
580
+ skipped.update(errs)
581
+ if not events and cur == prev:
582
+ continue
583
+
584
+ def advance_watermark(p: str = path, c: dict[str, Any] = cur) -> None:
585
+ chats[p] = c
586
+
587
+ yield events, advance_watermark
588
+ unreadable += yield from _sdk_batches(cfg.home, sdk)
589
+ warn_skipped(NAME, skipped, unaffected="human")
590
+ if unreadable:
591
+ raise CollectorUnavailable(f"{NAME}: {unreadable} store(s) locked or corrupt, left for the "
592
+ "next scan")
@@ -0,0 +1,95 @@
1
+ """Cursor-SDK collector: token usage from SDK-driven agents that persist a run manifest.
2
+
3
+ Agents built on the Cursor Agent SDK don't write the CLI/IDE logs the other collectors read
4
+ - the SDK hands per-run usage back in-process and the calling app saves it. The
5
+ research-backend-tickets-orchestrator is the reference case: it writes a per-ticket
6
+ `manifest.json` under `<root>/.state/runs/<TICKET>/` whose `runs[]` each carry
7
+ `{input_tokens, output_tokens, total_tokens, cache_read_tokens}` (real shape; `total` already
8
+ sums input+output+cache_read) plus the `model` and `phase`.
9
+
10
+ Point config `cursor_sdk_manifests` at a glob of those manifest files (e.g.
11
+ `~/proj/**/.state/runs/*/manifest.json`); unset -> this collector no-ops. Like the `cursor`
12
+ collector we FULL-SCAN every manifest each run and dedup at the sink via `event_id` +
13
+ ON CONFLICT: the manifest file is small and rewritten in place, but a run's `usage` is written
14
+ once at finish and never changes, so re-emitting the same run is idempotent.
15
+
16
+ LOWER BOUND: the orchestrator deletes a ticket's transcripts when its PR merges, so a ticket
17
+ finished-and-merged between two scans is invisible. Runs are scoped to the manifest schema,
18
+ so unrelated JSON matched by the glob is ignored (no `runs` list -> skipped).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import glob
24
+ import json
25
+ import os
26
+ from collections import Counter
27
+ from typing import Any, Callable, Iterator
28
+
29
+ from ..config import Config
30
+ from ..event import UsageEvent, make_event_id
31
+ from ._util import noop_advance, project_name, raise_if_unreadable, safe_int, warn_skipped
32
+
33
+ NAME = "cursor_sdk"
34
+
35
+
36
+ def _run_event(r: object, pos: int, ticket: object, project: str | None) -> UsageEvent | None:
37
+ """One entry of runs[] as a usage row; None when it has no usage."""
38
+ if not isinstance(r, dict) or not isinstance(r.get("usage"), dict):
39
+ return None
40
+ u = r["usage"]
41
+ inp = safe_int(u.get("input_tokens"))
42
+ out = safe_int(u.get("output_tokens"))
43
+ cr = safe_int(u.get("cache_read_tokens"))
44
+ cw = safe_int(u.get("cache_write_tokens"))
45
+ total = safe_int(u.get("total_tokens")) or (inp + out + cr + cw)
46
+ if total == 0:
47
+ return None
48
+ # runs[] is append-only (never reordered/removed mid-list), so `pos` is a stable,
49
+ # unique per-manifest run id even when the optional `index`/`run_id` fields are absent.
50
+ idx = r.get("index")
51
+ run_disc = str(pos if idx is None else idx)
52
+ model = r.get("model")
53
+ return UsageEvent(
54
+ event_id=make_event_id(NAME, str(ticket), run_disc, str(r.get("run_id") or "")),
55
+ ts=r.get("finished_at") or r.get("started_at"),
56
+ agent=NAME,
57
+ model=model if isinstance(model, str) else None, # a list/dict is unhashable downstream
58
+ provider="cursor",
59
+ input_tokens=inp,
60
+ output_tokens=out,
61
+ cache_read_tokens=cr,
62
+ cache_write_tokens=cw,
63
+ total_tokens=total,
64
+ project=project,
65
+ )
66
+
67
+
68
+ def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[tuple[list[UsageEvent], Callable[[], None]]]:
69
+ pattern = cfg.cursor_sdk_manifests
70
+ if not pattern:
71
+ return
72
+ pattern = os.path.expanduser(pattern) # glob does not expand ~; the documented form uses it
73
+ events: list[UsageEvent] = []
74
+ skipped: Counter[str] = Counter()
75
+ for path in glob.glob(pattern, recursive=True):
76
+ try:
77
+ with open(path, encoding="utf-8") as fh:
78
+ m = json.load(fh)
79
+ except FileNotFoundError: # gone since the glob
80
+ continue
81
+ except OSError:
82
+ skipped["unreadable"] += 1 # a later scan re-reads it: this is a full scan
83
+ continue
84
+ except ValueError: # JSONDecodeError and UnicodeDecodeError
85
+ skipped["malformed"] += 1
86
+ continue
87
+ if not isinstance(m, dict) or not isinstance(m.get("runs"), list):
88
+ continue
89
+ ticket = m.get("ticket") or path
90
+ project = project_name(m.get("project")) # basename-only (never a full path) per CONTRACT.md
91
+ rows = (_run_event(r, pos, ticket, project) for pos, r in enumerate(m["runs"]))
92
+ events += [e for e in rows if e]
93
+ yield events, noop_advance
94
+ warn_skipped(NAME, skipped)
95
+ raise_if_unreadable(NAME, skipped)