agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Commit detection: which shell commands run `git commit`, what SHAs they printed, and which repo
|
|
2
|
+
they ran in."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from ..repo_keys import RepoKeys
|
|
10
|
+
|
|
11
|
+
# what a shell reads as a unit, so a separator inside it splits nothing: an escape, a closed quote,
|
|
12
|
+
# a comment at a word start; a quote that never closes matches nothing and stays literal text
|
|
13
|
+
_SHELL_TOKEN = re.compile(r"""\\.|'[^']*'|"(?:[^"\\]|\\.)*"|(?<![^\s;|&()])#[^\n]*"""
|
|
14
|
+
r"""|(?P<sep>&&|\|\||[;|\n])""", re.S)
|
|
15
|
+
_HEREDOC = re.compile(r"""<<-?\s*\\?(['"]?)([\w.-]+)\1([^\n]*)\n.*?\n\s*\2[ \t]*(?=\n|$)""", re.S)
|
|
16
|
+
# a double-quoted string that can hold "$(...)" with its own quotes: PATH="$(dirname "$(which x)")"
|
|
17
|
+
_DQ = r'"(?:[^"\\]|\\.)*"'
|
|
18
|
+
for _ in range(3): # shortcut: three levels of nesting; deeper falls back to the plain quote rule
|
|
19
|
+
_SUB = rf"""\$\((?:[^()'"\\]|\\.|'[^']*'|{_DQ}|\((?:[^()]|\([^()]*\))*\))*+\)"""
|
|
20
|
+
_DQ = rf'"(?:[^"\\$]|\\.|{_SUB}|\$)*+"'
|
|
21
|
+
_ARG = rf"""(?:[^\s"']|{_DQ}|"[^"]*"|'[^']*')++"""
|
|
22
|
+
# no text can match two alternatives of a repeated group here, and the loops below are possessive:
|
|
23
|
+
# a log line is untrusted, and overlap made matching time double with every extra wrapper
|
|
24
|
+
_WRAPPER = rf"(?:\w+={_ARG}|(?:[^\s=]*/)?env|command|timeout\s+\S+|rtk(?:\s+proxy)?)\s+"
|
|
25
|
+
# optional wrappers (VAR=x, env, command, timeout N, rtk [proxy]), then `git`, then only global
|
|
26
|
+
# options (-C <dir>, -c <k=v>, --flag[=v]), then the commit subcommand itself
|
|
27
|
+
GIT_COMMIT = re.compile(
|
|
28
|
+
rf"^\s*[({{]?\s*(?:{_WRAPPER})*+"
|
|
29
|
+
rf"(?:\S*/)?git(?:\s+(?:-[Cc]\s+{_ARG}|--[\w-]+(?:={_ARG})?|-\w+))*+\s+commit(?:\s|$)")
|
|
30
|
+
# `sh -c '...'`: its quoted argument is commands, unlike any other command's quoted argument
|
|
31
|
+
_SHELL_C = re.compile(rf"""^\s*[({{]?\s*(?:{_WRAPPER}|uv\s+run\s+)*+(?:\S*/)?(?:ba|z|da)?sh\s+"""
|
|
32
|
+
rf"""(?:-[\w-]+\s+)*?-\w*c\s+('[^']*'|"(?:[^"\\]|\\.)*")""", re.S)
|
|
33
|
+
# segments that print nothing, so a commit after only these still owns the output's first line
|
|
34
|
+
_SILENT = re.compile(rf"^\s*(?:(?:cd|pushd|set|export|true|mkdir|rm)\b|cat\s+>|\w+={_ARG}\s*$"
|
|
35
|
+
rf"|(?:\S*/)?git(?:\s+-C\s+{_ARG})?\s+add\b)")
|
|
36
|
+
_COMMIT_OUT = re.compile(r"^\[[^\]\s]+(?: \(root-commit\))? (?P<sha>[0-9a-f]{7,40})\]", re.M)
|
|
37
|
+
# The condensed form a command proxy prints instead of git's own "[branch sha]" line: "ok <sha>"
|
|
38
|
+
# or "<sha> <subject>" on the FIRST line. Most commits on a machine with such a proxy look like this.
|
|
39
|
+
_CONDENSED = re.compile(r"(?:ok )?(?P<sha>[0-9a-f]{7,40})(?:\s|$)")
|
|
40
|
+
_OK_LINE = re.compile(r"^ok (?P<sha>[0-9a-f]{7,40})\s*$", re.M)
|
|
41
|
+
# git's commit line with its branch and subject, for the link row (both are text: hashed or dropped)
|
|
42
|
+
COMMIT_LINE = re.compile(
|
|
43
|
+
r"^\[(?P<br>[^\]\s]+)(?: \(root-commit\))? [0-9a-f]{7,40}\] ?(?P<subj>[^\n]*)", re.M)
|
|
44
|
+
_CD = re.compile(rf"^\s*[({{]?\s*(?:cd|pushd)\s+({_ARG})\s*$")
|
|
45
|
+
_DASH_C = re.compile(rf"\s-C\s+({_ARG})")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _words(seg: str) -> list[str]:
|
|
49
|
+
"""A segment's words without its quoted strings and comment: a flag named there is no flag."""
|
|
50
|
+
return _SHELL_TOKEN.sub(lambda m: " " if m[0][0] in "'\"#" else m[0], seg).split()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _commit_match(seg: str) -> re.Match[str] | None:
|
|
54
|
+
"""GIT_COMMIT's match when the segment runs a real commit (not --dry-run)."""
|
|
55
|
+
g = GIT_COMMIT.match(seg)
|
|
56
|
+
return g if g and "--dry-run" not in _words(seg) else None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _expand(at: int, seg: str) -> list[tuple[int, str]]:
|
|
60
|
+
"""A shell wrapper's own segments when one of them commits, else the segment. A wrapper with no
|
|
61
|
+
commit stays whole: a cd inside it does not move the commands after it."""
|
|
62
|
+
s = _SHELL_C.match(seg)
|
|
63
|
+
if s:
|
|
64
|
+
inner = _segments(s[1][1:-1])
|
|
65
|
+
if any(_commit_match(x) for _, x in inner):
|
|
66
|
+
return [(at + s.start(1) + 1 + i, x) for i, x in inner]
|
|
67
|
+
return [(at, seg)]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _segments(cmd: str) -> list[tuple[int, str]]:
|
|
71
|
+
"""The non-blank command segments of `cmd`, each with where it starts. A heredoc body is data
|
|
72
|
+
(a file, a commit message), not commands: it is blanked, not dropped, so offsets hold in `cmd`.
|
|
73
|
+
A backslash-newline only joins lines, so it is blanked too."""
|
|
74
|
+
text = _HEREDOC.sub(lambda m: " " * (m.start(3) - m.start()) + m[3]
|
|
75
|
+
+ " " * (m.end() - m.end(3)), cmd)
|
|
76
|
+
text = _SHELL_TOKEN.sub(lambda m: " " if m[0] == "\\\n" else m[0], text)
|
|
77
|
+
segs, start = [], 0
|
|
78
|
+
for m in _SHELL_TOKEN.finditer(text):
|
|
79
|
+
if m["sep"]:
|
|
80
|
+
segs.append((start, text[start:m.start()]))
|
|
81
|
+
start = m.end()
|
|
82
|
+
segs.append((start, text[start:]))
|
|
83
|
+
return [e for at, x in segs if x.strip() for e in _expand(at, x)]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def commit_call(cmd: str) -> tuple[bool, bool] | None:
|
|
87
|
+
"""(is_amend, printed_before) when a command segment runs `git commit` (not --dry-run);
|
|
88
|
+
printed_before: a segment ahead of the commit may have printed, so the output's first line is
|
|
89
|
+
not necessarily the commit's."""
|
|
90
|
+
segs = [x for _, x in _segments(cmd)]
|
|
91
|
+
hits = [i for i, x in enumerate(segs) if _commit_match(x)]
|
|
92
|
+
if not hits:
|
|
93
|
+
return None
|
|
94
|
+
return (any("--amend" in _words(segs[i]) for i in hits),
|
|
95
|
+
not all(_SILENT.match(x) for x in segs[:hits[0]]))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def commit_args_at(cmd: str) -> int | None:
|
|
99
|
+
"""Where the arguments of the segment commit_call found start in `cmd`."""
|
|
100
|
+
for at, x in _segments(cmd):
|
|
101
|
+
g = _commit_match(x)
|
|
102
|
+
if g:
|
|
103
|
+
return at + g.end() - 1 # the space GIT_COMMIT ate; a -m search needs it
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def commit_dir(cmd: str) -> str | None:
|
|
108
|
+
"""Where the first commit call runs when the command moves there itself (`cd X && git commit`,
|
|
109
|
+
`git -C X commit`), relative to the record's cwd unless absolute; None when it does not move
|
|
110
|
+
or the move is not knowable (a variable, a command substitution)."""
|
|
111
|
+
d = None
|
|
112
|
+
for _, x in _segments(cmd):
|
|
113
|
+
m, g = _CD.match(x), _commit_match(x)
|
|
114
|
+
# a cd moves the shell for the segments after it; a -C only the commit itself
|
|
115
|
+
if m:
|
|
116
|
+
args = [m[1]]
|
|
117
|
+
elif g:
|
|
118
|
+
args = [c[1] for c in _DASH_C.finditer(g[0])]
|
|
119
|
+
else:
|
|
120
|
+
continue
|
|
121
|
+
for arg in args:
|
|
122
|
+
if "$" in arg or "`" in arg:
|
|
123
|
+
return None
|
|
124
|
+
d = os.path.join(d or "", os.path.expanduser(arg.strip("\"'")))
|
|
125
|
+
if g:
|
|
126
|
+
return d
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def commit_repo(keys: RepoKeys, cwd: object, target: object, fallback: str | None) -> str | None:
|
|
131
|
+
"""A commit's repo_key: the directory the command moved to, else the record's cwd, else
|
|
132
|
+
`fallback` (the worktree's original checkout, for a cwd that is gone). A relative target
|
|
133
|
+
with no cwd is not resolved: it would be relative to aut's own directory."""
|
|
134
|
+
if isinstance(target, str) and (isinstance(cwd, str) or os.path.isabs(target)):
|
|
135
|
+
k = keys.of_cwd(os.path.join(cwd, target) if isinstance(cwd, str) else target)
|
|
136
|
+
if k:
|
|
137
|
+
return k
|
|
138
|
+
return keys.of_cwd(cwd) or fallback
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def commit_shas(text: str, err: bool, before: bool, structured: str | None = None) -> list[str]:
|
|
142
|
+
"""SHAs a commit call printed: git's `[branch sha]` lines; failing that, for a successful
|
|
143
|
+
call only, the agent's structured sha, else the condensed proxy form where it can be the
|
|
144
|
+
commit's own (the first line, or the last `ok <sha>` when an earlier segment may have
|
|
145
|
+
printed)."""
|
|
146
|
+
found = [m["sha"] for m in _COMMIT_OUT.finditer(text)]
|
|
147
|
+
if found or err:
|
|
148
|
+
return found
|
|
149
|
+
if structured:
|
|
150
|
+
return [structured]
|
|
151
|
+
if before:
|
|
152
|
+
return [m["sha"] for m in _OK_LINE.finditer(text)][-1:]
|
|
153
|
+
m = _CONDENSED.match(text.split("\n", 1)[0])
|
|
154
|
+
return [m["sha"]] if m else []
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Cursor collector (EXPERIMENTAL): local state.vscdb SQLite.
|
|
2
|
+
|
|
3
|
+
Cursor stores per-message rows in cursorDiskKV under `bubbleId:<composerId>:<bubbleId>`,
|
|
4
|
+
each with a tokenCount {inputTokens, outputTokens} and sometimes a usageUuid. Reality
|
|
5
|
+
checks from real data:
|
|
6
|
+
- the model id is NOT stored on the bubble (dashboard API would have it),
|
|
7
|
+
- most token bubbles carry no timestamp on the bubble itself, but joining to the
|
|
8
|
+
`composerData:<composerId>` row recovers a createdAt for ~100% of them.
|
|
9
|
+
|
|
10
|
+
Because the DB is small and Cursor can write a bubble's tokenCount AFTER first insert,
|
|
11
|
+
we full-scan every run (no watermark) and rely on event_id + ON CONFLICT to dedup at the
|
|
12
|
+
sink. This is a LOWER BOUND: sessions with no local snapshot are invisible. For complete,
|
|
13
|
+
model-attributed, timestamped data, use the `cursor_api` collector, with `cursor_human_only` here
|
|
14
|
+
to keep the prompts without double-counting the tokens.
|
|
15
|
+
|
|
16
|
+
Known limitation: the first NONZERO tokenCount seen for a bubble wins. If Cursor later
|
|
17
|
+
rewrites that bubble to a different nonzero count, the sink's ON CONFLICT DO NOTHING keeps
|
|
18
|
+
the original (a 0 -> nonzero fill-in is fine, since zero rows are skipped). Rare in practice.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import sqlite3
|
|
26
|
+
import sys
|
|
27
|
+
from collections import Counter
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any, Callable, Iterator
|
|
30
|
+
|
|
31
|
+
from ..config import Config
|
|
32
|
+
from ..event import CollectorUnavailable, UsageEvent, make_event_id
|
|
33
|
+
from .activity_common import human_event
|
|
34
|
+
from ._util import (MALFORMED_RECORD, count_field, dict_or_empty, epoch_ms_to_iso, noop_advance,
|
|
35
|
+
warn_skipped)
|
|
36
|
+
|
|
37
|
+
NAME = "cursor"
|
|
38
|
+
# Cursor stores state.vscdb in the OS-specific app-data dir. Subpath is the same everywhere.
|
|
39
|
+
_CURSOR_SUBPATH = "Cursor/User/globalStorage/state.vscdb"
|
|
40
|
+
_BATCH = 500
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def cursor_db(cfg: Config) -> Path:
|
|
44
|
+
"""Locate Cursor's state.vscdb across macOS / Windows / Linux, or an explicit override."""
|
|
45
|
+
if cfg.cursor_db_path:
|
|
46
|
+
return Path(cfg.cursor_db_path).expanduser()
|
|
47
|
+
home = cfg.home
|
|
48
|
+
if sys.platform == "darwin":
|
|
49
|
+
return home / "Library" / "Application Support" / _CURSOR_SUBPATH
|
|
50
|
+
if sys.platform == "win32":
|
|
51
|
+
base = os.environ.get("APPDATA")
|
|
52
|
+
return (Path(base) if base else home / "AppData" / "Roaming") / _CURSOR_SUBPATH
|
|
53
|
+
return home / ".config" / _CURSOR_SUBPATH # linux + other
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def connect_ro(path: object) -> sqlite3.Connection | None:
|
|
57
|
+
"""Read-only connect, safe alongside a running Cursor (WAL allows concurrent readers)."""
|
|
58
|
+
try:
|
|
59
|
+
con = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5.0)
|
|
60
|
+
con.execute("PRAGMA query_only = ON")
|
|
61
|
+
return con
|
|
62
|
+
except sqlite3.DatabaseError: # locked, corrupt, or not-a-database file
|
|
63
|
+
return None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _model(bubble: dict[str, Any]) -> str | None:
|
|
67
|
+
for k in ("model", "modelType", "modelName"):
|
|
68
|
+
v = bubble.get(k)
|
|
69
|
+
if isinstance(v, str) and v:
|
|
70
|
+
return v
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _ts(bubble: dict[str, Any], meta: dict[str, Any]) -> str | None:
|
|
75
|
+
"""Bubble createdAt is an ISO string; composerData createdAt is epoch-ms. Handle both."""
|
|
76
|
+
bts = bubble.get("createdAt")
|
|
77
|
+
if isinstance(bts, str) and bts:
|
|
78
|
+
return bts
|
|
79
|
+
return epoch_ms_to_iso(bts or meta.get("ts"))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _composers(cur: sqlite3.Cursor, skipped: Counter[str]) -> tuple[dict[str, dict[str, Any]],
|
|
83
|
+
sqlite3.DatabaseError | None]:
|
|
84
|
+
"""composerId -> {ts, model}: recovers a timestamp for bubbles that lack one. Also the read
|
|
85
|
+
failure, if any: the bubbles still go out, and some lose their ts or model."""
|
|
86
|
+
comp: dict[str, dict[str, Any]] = {}
|
|
87
|
+
try:
|
|
88
|
+
for key, val in cur.execute(
|
|
89
|
+
"SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%'"
|
|
90
|
+
):
|
|
91
|
+
if val is None:
|
|
92
|
+
skipped["missing_payloads"] += 1
|
|
93
|
+
continue
|
|
94
|
+
try:
|
|
95
|
+
o = json.loads(val)
|
|
96
|
+
except (ValueError, TypeError):
|
|
97
|
+
o = None
|
|
98
|
+
if not isinstance(o, dict):
|
|
99
|
+
skipped["malformed"] += 1
|
|
100
|
+
continue
|
|
101
|
+
comp[key.split(":", 1)[1]] = {
|
|
102
|
+
# createdAt only (stable). lastUpdatedAt changes on every composer update,
|
|
103
|
+
# which would shift the event ts and defeat ON CONFLICT (event_id, ts).
|
|
104
|
+
"ts": o.get("createdAt"),
|
|
105
|
+
"model": _model(o),
|
|
106
|
+
}
|
|
107
|
+
except sqlite3.DatabaseError as exc:
|
|
108
|
+
return comp, exc
|
|
109
|
+
return comp, None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _bubble_event(key: str, val: Any, comp: dict[str, dict[str, Any]], human_only: bool,
|
|
113
|
+
skipped: Counter[str]) -> UsageEvent | None:
|
|
114
|
+
"""The row of one bubble; None when it has none."""
|
|
115
|
+
if val is None:
|
|
116
|
+
skipped["missing_payloads"] += 1
|
|
117
|
+
return None
|
|
118
|
+
try:
|
|
119
|
+
o = json.loads(val)
|
|
120
|
+
if not isinstance(o, dict):
|
|
121
|
+
skipped["malformed"] += 1
|
|
122
|
+
return None
|
|
123
|
+
tc = dict_or_empty(o.get("tokenCount"))
|
|
124
|
+
inp = count_field(tc, "inputTokens")
|
|
125
|
+
out = count_field(tc, "outputTokens")
|
|
126
|
+
except MALFORMED_RECORD:
|
|
127
|
+
skipped["malformed"] += 1
|
|
128
|
+
return None
|
|
129
|
+
composer_id = key.split(":")[1]
|
|
130
|
+
bubble_id = o.get("bubbleId") or key.rsplit(":", 1)[-1]
|
|
131
|
+
meta = comp.get(composer_id) or {}
|
|
132
|
+
if o.get("type") == 1:
|
|
133
|
+
# A user bubble = a human prompt. No token count of its own, and no bubble timestamp
|
|
134
|
+
# either, so ts falls back to the composer's createdAt (COARSE: every prompt in a
|
|
135
|
+
# conversation shares one time). event_id is per-bubble.
|
|
136
|
+
return human_event(NAME, (composer_id, bubble_id), _ts(o, meta))
|
|
137
|
+
# Human-turns-only mode: the token rows are exactly what would double-count against
|
|
138
|
+
# cursor_api, and they are the inferior copy anyway (no cache attribution, no CLI usage). Drop
|
|
139
|
+
# them and keep the prompts, which cursor_api's feed does not carry at all.
|
|
140
|
+
if human_only or inp + out <= 0:
|
|
141
|
+
return None
|
|
142
|
+
uid = o.get("usageUuid") or f"{composer_id}:{bubble_id}"
|
|
143
|
+
return UsageEvent(
|
|
144
|
+
event_id=make_event_id(NAME, uid),
|
|
145
|
+
ts=_ts(o, meta),
|
|
146
|
+
agent=NAME,
|
|
147
|
+
model=_model(o) or meta.get("model"),
|
|
148
|
+
provider="cursor",
|
|
149
|
+
input_tokens=inp,
|
|
150
|
+
output_tokens=out,
|
|
151
|
+
total_tokens=inp + out,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[tuple[list[UsageEvent], Callable[[], None]]]:
|
|
156
|
+
db = cursor_db(cfg)
|
|
157
|
+
if not db.exists():
|
|
158
|
+
return
|
|
159
|
+
con = connect_ro(db)
|
|
160
|
+
if con is None:
|
|
161
|
+
raise CollectorUnavailable("cursor state.vscdb exists but could not be opened "
|
|
162
|
+
"(locked/corrupt); skipping cursor this run")
|
|
163
|
+
try:
|
|
164
|
+
cur = con.cursor()
|
|
165
|
+
skipped: Counter[str] = Counter()
|
|
166
|
+
comp, read_failure = _composers(cur, skipped)
|
|
167
|
+
try:
|
|
168
|
+
rows = cur.execute(
|
|
169
|
+
"SELECT key, value FROM cursorDiskKV WHERE key LIKE 'bubbleId:%'"
|
|
170
|
+
)
|
|
171
|
+
except sqlite3.DatabaseError as exc:
|
|
172
|
+
raise CollectorUnavailable(f"cursor bubbleId query failed ({exc}); skipping cursor") \
|
|
173
|
+
from exc
|
|
174
|
+
|
|
175
|
+
events: list[UsageEvent] = []
|
|
176
|
+
try:
|
|
177
|
+
for key, val in rows:
|
|
178
|
+
event = _bubble_event(key, val, comp, cfg.cursor_human_only, skipped)
|
|
179
|
+
if event is None:
|
|
180
|
+
continue
|
|
181
|
+
events.append(event)
|
|
182
|
+
if len(events) >= _BATCH:
|
|
183
|
+
yield events, noop_advance
|
|
184
|
+
events = []
|
|
185
|
+
except sqlite3.DatabaseError as exc: # corrupt DB can raise mid-iteration
|
|
186
|
+
read_failure = exc
|
|
187
|
+
yield events, noop_advance
|
|
188
|
+
warn_skipped(NAME, skipped)
|
|
189
|
+
if read_failure:
|
|
190
|
+
raise CollectorUnavailable(f"cursor row read failed ({read_failure}); emitted partial "
|
|
191
|
+
"data") from read_failure
|
|
192
|
+
finally:
|
|
193
|
+
con.close()
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Cursor usage-API collector (OPT-IN, network): accurate per-event token usage from Cursor's
|
|
2
|
+
own accounting, including CLI (`cursor-agent`) and SDK usage the local state.vscdb never records.
|
|
3
|
+
|
|
4
|
+
The local `cursor` collector is a lower bound (IDE bubbles only, no model/cache attribution) and
|
|
5
|
+
the Cursor CLI stores no token counts on disk at all. Cursor's dashboard, however, has an exact
|
|
6
|
+
per-event feed. This collector reads the local Cursor auth token (never stored, read at run time),
|
|
7
|
+
resolves the signed-in user, and pulls that user's OWN usage events.
|
|
8
|
+
|
|
9
|
+
Enable with `cursor_api = true` in config; OFF by default because it makes authenticated network
|
|
10
|
+
calls with your Cursor session token. Reads the token from the same state.vscdb the `cursor`
|
|
11
|
+
collector uses (or `cursor_db_path`); if Cursor is not installed, it no-ops.
|
|
12
|
+
|
|
13
|
+
PRIVACY / MULTI-USER SAFETY: on a team plan the events feed is team-scoped. This collector scopes
|
|
14
|
+
the query to the signed-in user's own userId AND re-checks `owningUser == userId` on every event
|
|
15
|
+
before emitting, so a teammate's usage can never be captured even if server-side scoping changed.
|
|
16
|
+
No prompt/response content is read - only token counts, model, and timestamp.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import base64
|
|
22
|
+
import json
|
|
23
|
+
import sqlite3
|
|
24
|
+
import time
|
|
25
|
+
import urllib.error
|
|
26
|
+
import urllib.parse
|
|
27
|
+
import urllib.request
|
|
28
|
+
from typing import Any, Callable, Iterator
|
|
29
|
+
|
|
30
|
+
from ..config import Config
|
|
31
|
+
from ..event import CollectorUnavailable, UsageEvent, make_event_id
|
|
32
|
+
from ..net import HTTP_ERRORS, NoRedirect
|
|
33
|
+
from ._util import MS_PER_S, epoch_ms_to_iso, safe_int
|
|
34
|
+
from .cursor import connect_ro, cursor_db
|
|
35
|
+
|
|
36
|
+
NAME = "cursor_api" # registry / config key + event_id namespace (internal)
|
|
37
|
+
_AGENT = "cursor" # emitted agent label: this IS the accurate Cursor source, so it reads as
|
|
38
|
+
# plain "cursor" in Grafana + reports. The local `cursor` collector emits
|
|
39
|
+
# the same label: run one token source, or set cursor_human_only = true.
|
|
40
|
+
_ORIGIN = "https://cursor.com"
|
|
41
|
+
_BASE = f"{_ORIGIN}/api/dashboard"
|
|
42
|
+
_DAY_MS = 86_400 * MS_PER_S
|
|
43
|
+
_PAGE = 100
|
|
44
|
+
_MAX_PAGES = 200 # hard stop (200*100 = 20k events/window) so a bug can't loop forever
|
|
45
|
+
_WINDOW_MS = _DAY_MS # a gap is read oldest first, one day per request window
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _auth(cfg: Config) -> tuple[str, int] | None:
|
|
49
|
+
"""(session-cookie, teamId) from the local Cursor state DB, or None if unavailable."""
|
|
50
|
+
db = cursor_db(cfg)
|
|
51
|
+
if not db.exists():
|
|
52
|
+
return None
|
|
53
|
+
con = connect_ro(db)
|
|
54
|
+
if con is None:
|
|
55
|
+
return None
|
|
56
|
+
try:
|
|
57
|
+
def item(k: str) -> Any:
|
|
58
|
+
row = con.execute("SELECT value FROM ItemTable WHERE key=?", (k,)).fetchone()
|
|
59
|
+
return row[0] if row else None
|
|
60
|
+
token = item("cursorAuth/accessToken")
|
|
61
|
+
team_raw = item("cursorAuth/cachedTeam")
|
|
62
|
+
if not token:
|
|
63
|
+
return None
|
|
64
|
+
# sub is the second JWT segment's "sub"; the web cookie is <sub>::<jwt> (:: url-encoded)
|
|
65
|
+
payload = token.split(".")[1]
|
|
66
|
+
payload += "=" * (-len(payload) % 4)
|
|
67
|
+
claims = json.loads(base64.urlsafe_b64decode(payload))
|
|
68
|
+
if not isinstance(claims, dict): # a JWT payload that decodes to a list/scalar
|
|
69
|
+
return None
|
|
70
|
+
sub = claims.get("sub", "")
|
|
71
|
+
try:
|
|
72
|
+
team = json.loads(team_raw) if team_raw else {}
|
|
73
|
+
team_id = int(team.get("teamId") or 0) if isinstance(team, dict) else 0
|
|
74
|
+
except (TypeError, ValueError):
|
|
75
|
+
team_id = 0
|
|
76
|
+
cookie = f"WorkosCursorSessionToken={urllib.parse.quote(sub)}%3A%3A{token}"
|
|
77
|
+
return cookie, team_id
|
|
78
|
+
except (sqlite3.DatabaseError, ValueError, KeyError, IndexError, TypeError, AttributeError):
|
|
79
|
+
return None # malformed/partial token -> treat as no auth, never crash
|
|
80
|
+
finally:
|
|
81
|
+
con.close()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _post(cookie: str, name: str, body: dict[str, Any]) -> Any:
|
|
85
|
+
"""POST a dashboard endpoint. Origin/Referer clear Cursor's CSRF ('Invalid origin') guard.
|
|
86
|
+
A redirect is not followed: the session cookie must reach cursor.com and nowhere else.
|
|
87
|
+
Raises one of HTTP_ERRORS on transport/HTTP failure; the caller decides how to handle it."""
|
|
88
|
+
req = urllib.request.Request(
|
|
89
|
+
f"{_BASE}/{name}", method="POST", data=json.dumps(body).encode(),
|
|
90
|
+
headers={"Cookie": cookie, "Content-Type": "application/json",
|
|
91
|
+
"User-Agent": "Mozilla/5.0", "Origin": _ORIGIN,
|
|
92
|
+
"Referer": f"{_ORIGIN}/dashboard"})
|
|
93
|
+
with urllib.request.build_opener(NoRedirect).open(req, timeout=30) as resp:
|
|
94
|
+
return json.loads(resp.read())
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _same_user(owner: object, uid: object) -> bool:
|
|
98
|
+
"""True only when BOTH ids are present, non-empty SCALARS and equal. Guards the privacy
|
|
99
|
+
boundary: a missing/empty owningUser must never match an empty/'None' uid, and a container
|
|
100
|
+
(list/dict) must never match another equal container via str() coercion."""
|
|
101
|
+
if not isinstance(owner, (str, int)) or not isinstance(uid, (str, int)):
|
|
102
|
+
return False
|
|
103
|
+
if isinstance(owner, bool) or isinstance(uid, bool): # bool is an int subclass
|
|
104
|
+
return False
|
|
105
|
+
o, u = str(owner).strip(), str(uid).strip()
|
|
106
|
+
return o != "" and u != "" and o.lower() != "none" and o == u
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _map_event(e: dict[str, Any], uid: object) -> UsageEvent | None:
|
|
110
|
+
"""One Cursor usage event -> UsageEvent, or None if it is not this user's or has no tokens."""
|
|
111
|
+
if not _same_user(e.get("owningUser"), uid):
|
|
112
|
+
return None # SAFETY: never emit another team member's event
|
|
113
|
+
tu = e.get("tokenUsage")
|
|
114
|
+
if not isinstance(tu, dict):
|
|
115
|
+
return None
|
|
116
|
+
inp = safe_int(tu.get("inputTokens"))
|
|
117
|
+
out = safe_int(tu.get("outputTokens"))
|
|
118
|
+
cr = safe_int(tu.get("cacheReadTokens"))
|
|
119
|
+
cw = safe_int(tu.get("cacheWriteTokens"))
|
|
120
|
+
total = inp + out + cr + cw
|
|
121
|
+
if total == 0:
|
|
122
|
+
return None
|
|
123
|
+
ts_ms = e.get("timestamp")
|
|
124
|
+
conv = str(e.get("conversationId") or "")
|
|
125
|
+
model = e.get("model")
|
|
126
|
+
if not isinstance(model, str):
|
|
127
|
+
model = None
|
|
128
|
+
return UsageEvent(
|
|
129
|
+
# no per-event id in the feed; (user, ts, conversation, model) is unique per event
|
|
130
|
+
event_id=make_event_id(NAME, str(uid), str(ts_ms), conv, model or ""),
|
|
131
|
+
ts=epoch_ms_to_iso(ts_ms),
|
|
132
|
+
agent=_AGENT,
|
|
133
|
+
model=model,
|
|
134
|
+
provider="cursor",
|
|
135
|
+
input_tokens=inp,
|
|
136
|
+
output_tokens=out,
|
|
137
|
+
cache_read_tokens=cr,
|
|
138
|
+
cache_write_tokens=cw,
|
|
139
|
+
total_tokens=total,
|
|
140
|
+
project=None,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def describe(cfg: Config, wm: dict[str, Any]) -> str:
|
|
145
|
+
if "since_ms" in wm:
|
|
146
|
+
return f"since_ms={wm['since_ms']}"
|
|
147
|
+
return "enabled, not yet scanned" if cfg.cursor_api else "disabled (cursor_api=false)"
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _page_events(batch: list[Any], uid: object, since: int) -> tuple[list[UsageEvent], int, int]:
|
|
151
|
+
"""(this user's events, how many rows were AT/AFTER the watermark, the newest of the events)."""
|
|
152
|
+
events: list[UsageEvent] = []
|
|
153
|
+
fresh_rows, newest = 0, since
|
|
154
|
+
for raw in batch:
|
|
155
|
+
if not isinstance(raw, dict): # a stray row must not fail the page
|
|
156
|
+
continue
|
|
157
|
+
ms = safe_int(raw.get("timestamp"))
|
|
158
|
+
# SKIP (not break) rows older than the watermark: robust to a page that isn't perfectly
|
|
159
|
+
# newest-first, so a stray old row can't hide newer rows behind it.
|
|
160
|
+
if ms < since:
|
|
161
|
+
continue
|
|
162
|
+
fresh_rows += 1 # count BEFORE _map_event so a page of teammate/zero rows
|
|
163
|
+
ev = _map_event(raw, uid) # (dropped here) doesn't look like "nothing new" and halt.
|
|
164
|
+
if ev is not None:
|
|
165
|
+
newest = max(newest, ms)
|
|
166
|
+
events.append(ev)
|
|
167
|
+
return events, fresh_rows, newest
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _fetch_events(cookie: str, team_id: int, uid: object, since: int, start: int,
|
|
171
|
+
end: int) -> tuple[list[UsageEvent], int]:
|
|
172
|
+
"""Read a complete window, splitting on saturation; an irreducible cap refuses an ACK."""
|
|
173
|
+
events: list[UsageEvent] = []
|
|
174
|
+
newest = since
|
|
175
|
+
for page in range(1, _MAX_PAGES + 1):
|
|
176
|
+
# Boundary events stay inside the request whether the API uses inclusive or strict dates.
|
|
177
|
+
body = {"teamId": team_id, "userId": uid, "page": page, "pageSize": _PAGE,
|
|
178
|
+
"startDate": start - 1, "endDate": end + 1}
|
|
179
|
+
data = _post(cookie, "get-filtered-usage-events", body)
|
|
180
|
+
batch = data.get("usageEventsDisplay") if isinstance(data, dict) else None
|
|
181
|
+
# Cursor omits every default field from its empty ProtoJSON response.
|
|
182
|
+
if isinstance(data, dict) and not data:
|
|
183
|
+
batch = []
|
|
184
|
+
if not isinstance(batch, list):
|
|
185
|
+
raise CollectorUnavailable("cursor_api: invalid usage page; watermark unchanged")
|
|
186
|
+
bounded = [raw for raw in batch if isinstance(raw, dict)
|
|
187
|
+
and start <= safe_int(raw.get("timestamp")) <= end]
|
|
188
|
+
got, _, page_newest = _page_events(bounded, uid, since)
|
|
189
|
+
events += got
|
|
190
|
+
newest = max(newest, page_newest)
|
|
191
|
+
if len(batch) < _PAGE:
|
|
192
|
+
return events, newest
|
|
193
|
+
if start >= end:
|
|
194
|
+
raise CollectorUnavailable("cursor_api: page cap reached within one millisecond; "
|
|
195
|
+
"cannot prove completeness, watermark unchanged")
|
|
196
|
+
events.clear()
|
|
197
|
+
middle = (start + end) // 2
|
|
198
|
+
left, left_newest = _fetch_events(cookie, team_id, uid, since, start, middle)
|
|
199
|
+
right, right_newest = _fetch_events(cookie, team_id, uid, since, middle + 1, end)
|
|
200
|
+
return left + right, max(left_newest, right_newest)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def collect(cfg: Config, wm: dict[str, Any]) -> Iterator[tuple[list[UsageEvent], Callable[[], None]]]:
|
|
204
|
+
if not cfg.cursor_api:
|
|
205
|
+
return
|
|
206
|
+
auth = _auth(cfg)
|
|
207
|
+
if auth is None:
|
|
208
|
+
if cursor_db(cfg).exists():
|
|
209
|
+
raise CollectorUnavailable("cursor_api: no usable token in Cursor's state DB (signed "
|
|
210
|
+
"out?); skipping this run")
|
|
211
|
+
return
|
|
212
|
+
cookie, team_id = auth
|
|
213
|
+
try:
|
|
214
|
+
me = _post(cookie, "get-me", {})
|
|
215
|
+
uid = me.get("userId") if isinstance(me, dict) else None
|
|
216
|
+
if not uid: # None, empty, or non-dict response -> can't self-scope, skip
|
|
217
|
+
raise CollectorUnavailable("cursor_api: could not resolve a userId; skipping")
|
|
218
|
+
since = safe_int(wm.get("since_ms"))
|
|
219
|
+
# first run (no watermark): bounded lookback window instead of all history
|
|
220
|
+
lo = since if since else _now_ms() - cfg.cursor_api_lookback_days * _DAY_MS
|
|
221
|
+
# A gap is walked oldest first, one window at a time, and each window moves the watermark
|
|
222
|
+
# once delivered: a month away can hold more events than one request's page cap.
|
|
223
|
+
while True:
|
|
224
|
+
now = _now_ms()
|
|
225
|
+
hi = min(lo + _WINDOW_MS, now)
|
|
226
|
+
events, newest = _fetch_events(cookie, team_id, uid, lo, lo, hi)
|
|
227
|
+
# A closed window moves the watermark to its end (the next starts there, so an event
|
|
228
|
+
# exactly on the edge is read twice and deduped). The open window ends at the newest
|
|
229
|
+
# event, as before, so an event Cursor records late is still picked up.
|
|
230
|
+
mark = hi if hi < now else newest
|
|
231
|
+
|
|
232
|
+
def advance_watermark(w: dict[str, Any] = wm, n: int = mark) -> None:
|
|
233
|
+
if n:
|
|
234
|
+
w["since_ms"] = n
|
|
235
|
+
|
|
236
|
+
yield events, advance_watermark
|
|
237
|
+
if hi >= now:
|
|
238
|
+
return
|
|
239
|
+
lo = hi
|
|
240
|
+
except (*HTTP_ERRORS, json.JSONDecodeError, UnicodeDecodeError, AttributeError,
|
|
241
|
+
TypeError) as exc:
|
|
242
|
+
raise CollectorUnavailable(f"cursor_api: usage fetch failed ({exc}); skipping this run") \
|
|
243
|
+
from exc
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _now_ms() -> int:
|
|
247
|
+
return int(time.time() * MS_PER_S)
|