agent-decoder 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_decoder-0.3.2.dist-info/METADATA +387 -0
- agent_decoder-0.3.2.dist-info/RECORD +89 -0
- agent_decoder-0.3.2.dist-info/WHEEL +4 -0
- agent_decoder-0.3.2.dist-info/entry_points.txt +2 -0
- agent_decoder-0.3.2.dist-info/licenses/LICENSE +111 -0
- clau_decode/__init__.py +8 -0
- clau_decode/__main__.py +11 -0
- clau_decode/_auth_env.py +118 -0
- clau_decode/_pty_preexec.py +42 -0
- clau_decode/analytics/__init__.py +0 -0
- clau_decode/analytics/aggregator.py +55 -0
- clau_decode/analytics/cost.py +76 -0
- clau_decode/analytics/extractor.py +32 -0
- clau_decode/analytics/fast.py +289 -0
- clau_decode/analytics/models.py +49 -0
- clau_decode/analytics/pricing.py +162 -0
- clau_decode/analytics/prompt.py +34 -0
- clau_decode/analytics/service.py +23 -0
- clau_decode/analytics/stats.py +117 -0
- clau_decode/analytics/tips.py +176 -0
- clau_decode/btw_capture.py +290 -0
- clau_decode/cli.py +504 -0
- clau_decode/config.py +144 -0
- clau_decode/db.py +2054 -0
- clau_decode/driver_manager.py +387 -0
- clau_decode/drivers/__init__.py +27 -0
- clau_decode/drivers/base.py +170 -0
- clau_decode/drivers/registry.py +113 -0
- clau_decode/drivers/tmux_driver.py +488 -0
- clau_decode/editor.py +182 -0
- clau_decode/events_bus.py +69 -0
- clau_decode/locks.py +388 -0
- clau_decode/migrate.py +1313 -0
- clau_decode/models.py +285 -0
- clau_decode/parser.py +466 -0
- clau_decode/providers/__init__.py +30 -0
- clau_decode/providers/base.py +132 -0
- clau_decode/providers/claude.py +81 -0
- clau_decode/providers/codex.py +441 -0
- clau_decode/providers/registry.py +89 -0
- clau_decode/pty_native.py +39 -0
- clau_decode/pty_runner.py +2406 -0
- clau_decode/pty_screen_state.py +65 -0
- clau_decode/recap_runner.py +230 -0
- clau_decode/reporter.py +305 -0
- clau_decode/scanner.py +125 -0
- clau_decode/server.py +2841 -0
- clau_decode/static/android-chrome-192x192.png +0 -0
- clau_decode/static/android-chrome-512x512.png +0 -0
- clau_decode/static/app-icon.png +0 -0
- clau_decode/static/apple-touch-icon.png +0 -0
- clau_decode/static/assets/ChatView-BBVXuITs.js +79 -0
- clau_decode/static/assets/ChatView-BrP-ENHg.css +1 -0
- clau_decode/static/assets/Dashboard-3CYO4HQw.js +1 -0
- clau_decode/static/assets/FileViewer-BwbDc3ky.js +1 -0
- clau_decode/static/assets/HelpPopup-CPtECjYw.js +1 -0
- clau_decode/static/assets/ScrollContainer-BT_Dzxah.js +1 -0
- clau_decode/static/assets/SearchOverlay-DqI7aZaC.js +1 -0
- clau_decode/static/assets/SessionItem-DlyKg80u.js +1 -0
- clau_decode/static/assets/SettingsModal-CZBfMqx7.js +1 -0
- clau_decode/static/assets/ShortcutsPopup-W7CPdCzW.js +1 -0
- clau_decode/static/assets/Sidebar-BBswhEua.js +1 -0
- clau_decode/static/assets/client-DTSXvAb8.js +1 -0
- clau_decode/static/assets/fira-code-DGosTW8U.woff2 +0 -0
- clau_decode/static/assets/formatRelative-j86webkr.js +1 -0
- clau_decode/static/assets/hljs-B3O74O7Z.js +35 -0
- clau_decode/static/assets/index-BHdCOIkq.css +10 -0
- clau_decode/static/assets/index-DfKltZ6z.js +5 -0
- clau_decode/static/assets/ioskeley-mono-C7VR2-sL.woff2 +0 -0
- clau_decode/static/assets/jetbrains-mono-V6pRDFza.woff2 +0 -0
- clau_decode/static/assets/julia-mono-Cbvba6sW.woff2 +0 -0
- clau_decode/static/assets/libertinus-mono-Coz17E-O.woff2 +0 -0
- clau_decode/static/assets/monaspace-argon-RS-2TGNg.woff2 +0 -0
- clau_decode/static/assets/nativePtyFonts-BaXKtUsn.js +1 -0
- clau_decode/static/assets/react-Bi_azaFt.js +9 -0
- clau_decode/static/assets/renderSnippet-CqpQQugH.js +1 -0
- clau_decode/static/assets/rolldown-runtime-S-ySWqyJ.js +1 -0
- clau_decode/static/assets/shared-DdVGprvF.js +1 -0
- clau_decode/static/assets/source-code-pro-DEXWLoHG.woff2 +0 -0
- clau_decode/static/assets/spline-sans-mono-mUpA6Mve.woff2 +0 -0
- clau_decode/static/assets/ui-BcGPjpYn.js +1 -0
- clau_decode/static/assets/xanh-mono-Dsg19vWS.woff2 +0 -0
- clau_decode/static/favicon-16x16.png +0 -0
- clau_decode/static/favicon-32x32.png +0 -0
- clau_decode/static/favicon.ico +0 -0
- clau_decode/static/index.html +24 -0
- clau_decode/static/site.webmanifest +1 -0
- clau_decode/update_check.py +51 -0
- clau_decode/watcher.py +63 -0
clau_decode/_auth_env.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Shared spawn-environment construction for claude-compatible binaries.
|
|
2
|
+
|
|
3
|
+
Centralises the "strip API-key vars or pass them through?" decision so all
|
|
4
|
+
PTY-backed spawn sites apply identical rules.
|
|
5
|
+
|
|
6
|
+
Decision matrix
|
|
7
|
+
---------------
|
|
8
|
+
|
|
9
|
+
``authMethod`` returned by ``<bin> auth status``:
|
|
10
|
+
|
|
11
|
+
- ``"claude.ai"`` — subscription auth. Strip ``ANTHROPIC_API_KEY`` and
|
|
12
|
+
``ANTHROPIC_AUTH_TOKEN`` from the spawn env, even if the user set them.
|
|
13
|
+
This is the original billing safeguard: a stale env var would silently
|
|
14
|
+
re-route traffic to an API account the user may not have intended to bill.
|
|
15
|
+
- ``"api_key"`` — pass everything through. The binary is self-aware that it
|
|
16
|
+
uses API key auth; both the env key and any other config it cares about
|
|
17
|
+
should reach the spawn.
|
|
18
|
+
- ``"none"`` / probe failure / unknown — pass everything through. The probe
|
|
19
|
+
is unreliable for env-based wrappers (e.g. cc-mirror's ``zai``, which
|
|
20
|
+
unsets the token before the probe runs, so the probe never sees it as
|
|
21
|
+
``api_key``). Stripping here would break the wrapper's only authentication
|
|
22
|
+
path. The trade-off: a user with ``ANTHROPIC_API_KEY`` exported and a
|
|
23
|
+
binary that reports ``"none"`` will route to that key — but that requires
|
|
24
|
+
the user to have explicitly exported the key, which is their signal.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import asyncio
|
|
30
|
+
import json as _json
|
|
31
|
+
import os
|
|
32
|
+
from typing import Final
|
|
33
|
+
|
|
34
|
+
_SUBSCRIPTION_BLOCKED_ENV: Final = frozenset(
|
|
35
|
+
{"ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"}
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# Session-identity vars the `claude` CLI sets on itself so a nested/child
|
|
39
|
+
# invocation can detect it's not a top-level session (e.g. it disables
|
|
40
|
+
# transcript saving). If clau-decode's own server process was launched from
|
|
41
|
+
# inside a Claude Code session, these leak into ``os.environ`` and — because
|
|
42
|
+
# spawn_env otherwise starts from a raw copy of it — get inherited by every
|
|
43
|
+
# native PTY clau-decode spawns, making an unrelated top-level `claude`
|
|
44
|
+
# session falsely believe it's a child of another one. Every PTY clau-decode
|
|
45
|
+
# spawns is its own top-level session, so these must never survive the copy.
|
|
46
|
+
_SESSION_IDENTITY_BLOCKED_ENV: Final = frozenset({"CLAUDE_CODE_CHILD_SESSION"})
|
|
47
|
+
|
|
48
|
+
_AUTH_PROBE_TIMEOUT_S: Final = 5.0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _strip_session_identity_env(env: dict[str, str]) -> dict[str, str]:
|
|
52
|
+
"""Remove Claude-Code session-identity markers that must never be inherited."""
|
|
53
|
+
for key in _SESSION_IDENTITY_BLOCKED_ENV:
|
|
54
|
+
env.pop(key, None)
|
|
55
|
+
return env
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _subscription_env() -> dict[str, str]:
|
|
59
|
+
"""Pure strip helper: ``os.environ`` minus the API-key vars.
|
|
60
|
+
|
|
61
|
+
Exposed for tests and as a primitive for ``spawn_env``. Direct callers
|
|
62
|
+
outside tests should prefer ``spawn_env(bin_name)`` so the strip is
|
|
63
|
+
gated by the binary's actual auth method.
|
|
64
|
+
"""
|
|
65
|
+
env = {
|
|
66
|
+
k: v for k, v in os.environ.items() if k not in _SUBSCRIPTION_BLOCKED_ENV
|
|
67
|
+
}
|
|
68
|
+
return _strip_session_identity_env(env)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
async def _bin_auth_method(bin_name: str) -> str:
|
|
72
|
+
"""Probe ``<bin_name> auth status`` for its ``authMethod`` string.
|
|
73
|
+
|
|
74
|
+
Returns the method name (``"claude.ai"`` / ``"api_key"`` / ``"none"``)
|
|
75
|
+
or ``""`` on any probe failure (spawn error, non-zero exit, JSON parse
|
|
76
|
+
failure, or timeout). Callers must treat ``""`` and ``"none"`` the same.
|
|
77
|
+
|
|
78
|
+
Inherits the full parent env so the probe can see any API key the user
|
|
79
|
+
has exported — some wrappers report ``"api_key"`` when a key is present
|
|
80
|
+
and ``"none"`` otherwise.
|
|
81
|
+
"""
|
|
82
|
+
try:
|
|
83
|
+
proc = await asyncio.create_subprocess_exec(
|
|
84
|
+
bin_name,
|
|
85
|
+
"auth",
|
|
86
|
+
"status",
|
|
87
|
+
stdout=asyncio.subprocess.PIPE,
|
|
88
|
+
stderr=asyncio.subprocess.PIPE,
|
|
89
|
+
)
|
|
90
|
+
except (FileNotFoundError, PermissionError):
|
|
91
|
+
return ""
|
|
92
|
+
try:
|
|
93
|
+
stdout, _stderr = await asyncio.wait_for(
|
|
94
|
+
proc.communicate(), timeout=_AUTH_PROBE_TIMEOUT_S
|
|
95
|
+
)
|
|
96
|
+
except asyncio.TimeoutError:
|
|
97
|
+
proc.kill()
|
|
98
|
+
return ""
|
|
99
|
+
if proc.returncode != 0:
|
|
100
|
+
return ""
|
|
101
|
+
try:
|
|
102
|
+
data = _json.loads(stdout.decode("utf-8", errors="replace"))
|
|
103
|
+
except _json.JSONDecodeError:
|
|
104
|
+
return ""
|
|
105
|
+
method = data.get("authMethod") if isinstance(data, dict) else None
|
|
106
|
+
return str(method) if isinstance(method, str) else ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def spawn_env(bin_name: str) -> dict[str, str]:
|
|
110
|
+
"""Build the spawn env for ``bin_name``.
|
|
111
|
+
|
|
112
|
+
See module docstring for the strip/keep decision matrix. The returned
|
|
113
|
+
dict starts from ``os.environ`` and is safe to mutate (it's a copy).
|
|
114
|
+
"""
|
|
115
|
+
auth_method = await _bin_auth_method(bin_name)
|
|
116
|
+
if auth_method == "claude.ai":
|
|
117
|
+
return _subscription_env()
|
|
118
|
+
return _strip_session_identity_env(dict(os.environ))
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""PTY controlling-TTY attach wrapper.
|
|
2
|
+
|
|
3
|
+
uvloop's subprocess implementation does not run ``preexec_fn`` for safety
|
|
4
|
+
reasons (libuv constraint). The PTY runner needs ``setsid()`` +
|
|
5
|
+
``TIOCSCTTY`` on the slave fd before the spawned ``claude`` binary's main()
|
|
6
|
+
runs, or the TUI will fail to render.
|
|
7
|
+
|
|
8
|
+
Solution: invoke this wrapper as the entry point. It performs the
|
|
9
|
+
controlling-TTY claim in the *child* process (after Python's subprocess
|
|
10
|
+
infrastructure has dup2'd the slave fd into fd 0/1/2), then ``execvp`` into
|
|
11
|
+
the real target binary. From here on the child is fully detached and runs
|
|
12
|
+
under its own session with the slave as its controlling TTY.
|
|
13
|
+
|
|
14
|
+
Usage:
|
|
15
|
+
python3 -m clau_decode._pty_preexec <bin_name> [args...]
|
|
16
|
+
|
|
17
|
+
The first argv after the module name is the target binary; remaining args
|
|
18
|
+
are forwarded.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import fcntl
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import termios
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main() -> None:
|
|
30
|
+
if len(sys.argv) < 2:
|
|
31
|
+
sys.stderr.write("usage: _pty_preexec <binary> [args...]\n")
|
|
32
|
+
raise SystemExit(2)
|
|
33
|
+
|
|
34
|
+
os.setsid()
|
|
35
|
+
fcntl.ioctl(0, termios.TIOCSCTTY, 0)
|
|
36
|
+
|
|
37
|
+
target = sys.argv[1]
|
|
38
|
+
os.execvp(target, sys.argv[1:])
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
main()
|
|
File without changes
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from datetime import date, timezone
|
|
3
|
+
from clau_decode.models import Message
|
|
4
|
+
from .extractor import DeduplicatingExtractor, TokenExtractor
|
|
5
|
+
from .models import DailyBucket, TokenBreakdown
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SessionAggregator:
|
|
9
|
+
"""Aggregate token breakdowns across all messages in a session."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, extractor: TokenExtractor) -> None:
|
|
12
|
+
self._extractor = extractor
|
|
13
|
+
|
|
14
|
+
def aggregate(self, messages: list[Message]) -> TokenBreakdown:
|
|
15
|
+
dedup = DeduplicatingExtractor(self._extractor)
|
|
16
|
+
total = TokenBreakdown()
|
|
17
|
+
for msg in messages:
|
|
18
|
+
total = total + dedup.extract(msg)
|
|
19
|
+
return total
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DailyAggregator:
|
|
23
|
+
"""Bucket messages into UTC calendar days."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, extractor: TokenExtractor) -> None:
|
|
26
|
+
self._extractor = extractor
|
|
27
|
+
|
|
28
|
+
def aggregate(self, messages: list[Message]) -> list[DailyBucket]:
|
|
29
|
+
dedup = DeduplicatingExtractor(self._extractor)
|
|
30
|
+
by_id: dict[str, Message] = {m.id: m for m in messages}
|
|
31
|
+
buckets: dict[date, DailyBucket] = {}
|
|
32
|
+
seen_sessions: dict[date, set[str]] = {}
|
|
33
|
+
for msg in messages:
|
|
34
|
+
if msg.timestamp is None:
|
|
35
|
+
continue
|
|
36
|
+
day = msg.timestamp.astimezone(timezone.utc).date()
|
|
37
|
+
bd = dedup.extract(msg)
|
|
38
|
+
if bd.total == 0:
|
|
39
|
+
continue
|
|
40
|
+
if day not in buckets:
|
|
41
|
+
buckets[day] = DailyBucket(day=day)
|
|
42
|
+
seen_sessions[day] = set()
|
|
43
|
+
buckets[day].breakdown = buckets[day].breakdown + bd
|
|
44
|
+
seen_sessions[day].add(msg.session_id)
|
|
45
|
+
if (
|
|
46
|
+
msg.role == "assistant"
|
|
47
|
+
and not msg.is_sidechain
|
|
48
|
+
and not msg.is_meta
|
|
49
|
+
and msg.parent_id in by_id
|
|
50
|
+
and by_id[msg.parent_id].role == "user"
|
|
51
|
+
):
|
|
52
|
+
buckets[day].prompt_count += 1
|
|
53
|
+
for day, bucket in buckets.items():
|
|
54
|
+
bucket.session_count = len(seen_sessions[day])
|
|
55
|
+
return sorted(buckets.values(), key=lambda b: b.day)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from .models import PromptCost, TokenBreakdown
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from .pricing import CachedPricingStrategy, ModelPricing
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class SessionCost:
|
|
15
|
+
model: str
|
|
16
|
+
pricing: "ModelPricing | None"
|
|
17
|
+
breakdown: TokenBreakdown
|
|
18
|
+
input_usd: Decimal
|
|
19
|
+
output_usd: Decimal
|
|
20
|
+
cache_write_usd: Decimal
|
|
21
|
+
cache_read_usd: Decimal
|
|
22
|
+
total_usd: Decimal
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class MultiModelCost:
|
|
27
|
+
models: list[SessionCost]
|
|
28
|
+
total_usd: Decimal
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class CostEngine:
|
|
32
|
+
"""Compute USD cost for a model+breakdown pair using a pricing strategy."""
|
|
33
|
+
|
|
34
|
+
def __init__(self, strategy: "CachedPricingStrategy") -> None:
|
|
35
|
+
self._strategy = strategy
|
|
36
|
+
|
|
37
|
+
def compute(self, model: str, breakdown: TokenBreakdown) -> SessionCost:
|
|
38
|
+
pricing = self._strategy.get_pricing(model)
|
|
39
|
+
if pricing is None:
|
|
40
|
+
zero = Decimal("0")
|
|
41
|
+
return SessionCost(
|
|
42
|
+
model=model,
|
|
43
|
+
pricing=None,
|
|
44
|
+
breakdown=breakdown,
|
|
45
|
+
input_usd=zero,
|
|
46
|
+
output_usd=zero,
|
|
47
|
+
cache_write_usd=zero,
|
|
48
|
+
cache_read_usd=zero,
|
|
49
|
+
total_usd=zero,
|
|
50
|
+
)
|
|
51
|
+
M = Decimal("1000000")
|
|
52
|
+
inp = pricing.input_per_mtok * breakdown.input_tokens / M
|
|
53
|
+
out = pricing.output_per_mtok * breakdown.output_tokens / M
|
|
54
|
+
cw = pricing.cache_write_per_mtok * breakdown.cache_creation_tokens / M
|
|
55
|
+
cr = pricing.cache_read_per_mtok * breakdown.cache_read_tokens / M
|
|
56
|
+
return SessionCost(
|
|
57
|
+
model=model,
|
|
58
|
+
pricing=pricing,
|
|
59
|
+
breakdown=breakdown,
|
|
60
|
+
input_usd=inp,
|
|
61
|
+
output_usd=out,
|
|
62
|
+
cache_write_usd=cw,
|
|
63
|
+
cache_read_usd=cr,
|
|
64
|
+
total_usd=inp + out + cw + cr,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
def compute_multi(self, prompts: list[PromptCost]) -> MultiModelCost:
|
|
68
|
+
by_model: dict[str, TokenBreakdown] = {}
|
|
69
|
+
for p in prompts:
|
|
70
|
+
bd = by_model.setdefault(p.model, TokenBreakdown())
|
|
71
|
+
by_model[p.model] = bd + p.breakdown
|
|
72
|
+
results: list[SessionCost] = []
|
|
73
|
+
for model, bd in sorted(by_model.items()):
|
|
74
|
+
results.append(self.compute(model, bd))
|
|
75
|
+
total = sum((r.total_usd for r in results), Decimal("0"))
|
|
76
|
+
return MultiModelCost(models=results, total_usd=total)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from clau_decode.models import Message
|
|
3
|
+
from .models import TokenBreakdown
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TokenExtractor:
|
|
7
|
+
"""Extract a TokenBreakdown from a single Message."""
|
|
8
|
+
|
|
9
|
+
def extract(self, message: Message) -> TokenBreakdown:
|
|
10
|
+
if message.usage is None:
|
|
11
|
+
return TokenBreakdown()
|
|
12
|
+
u = message.usage
|
|
13
|
+
return TokenBreakdown(
|
|
14
|
+
input_tokens=u.input_tokens,
|
|
15
|
+
output_tokens=u.output_tokens,
|
|
16
|
+
cache_creation_tokens=u.cache_creation_input_tokens,
|
|
17
|
+
cache_read_tokens=u.cache_read_input_tokens,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DeduplicatingExtractor:
|
|
22
|
+
"""Wraps an extractor; returns zero breakdown for duplicate message IDs."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, inner: TokenExtractor) -> None:
|
|
25
|
+
self._inner = inner
|
|
26
|
+
self.seen_ids: set[str] = set()
|
|
27
|
+
|
|
28
|
+
def extract(self, message: Message) -> TokenBreakdown:
|
|
29
|
+
if message.id in self.seen_ids:
|
|
30
|
+
return TokenBreakdown()
|
|
31
|
+
self.seen_ids.add(message.id)
|
|
32
|
+
return self._inner.extract(message)
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""SQL-backed analytics over the full message corpus.
|
|
2
|
+
|
|
3
|
+
The Python scanners (``stats.py`` / ``service.py``) answer the dashboard's
|
|
4
|
+
all-message questions by loading every row — ``SELECT *`` pulls ~500 MB of
|
|
5
|
+
``content_json`` and ~145k rows become pydantic ``Message`` objects (~9 s) —
|
|
6
|
+
then scanning in memory. These functions compute the **identical shapes**
|
|
7
|
+
from data materialized at ingest:
|
|
8
|
+
|
|
9
|
+
- per-message token columns (``input_tokens`` … ``cache_read_tokens``)
|
|
10
|
+
- the ``message_blocks`` table (one row per tool_use / tool_result block,
|
|
11
|
+
with tool name, file path, and result char-count already extracted)
|
|
12
|
+
|
|
13
|
+
So the dashboard never reads ``content_json`` at query time — it aggregates
|
|
14
|
+
small indexed columns. Empirically ~1 s cold vs ~9 s, and the existing
|
|
15
|
+
signature cache keeps warm loads at ~3 ms.
|
|
16
|
+
|
|
17
|
+
Verified byte-for-byte against the scanners in
|
|
18
|
+
``tests/analytics/test_fast_parity.py``. Per-session/small-list analytics
|
|
19
|
+
keep using the scanners — these are only for the whole-corpus dashboard.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from clau_decode.models import Message, TokenUsage, ToolResultBlock, ToolUseBlock
|
|
27
|
+
|
|
28
|
+
from .stats import compute_stats
|
|
29
|
+
from .tips import (
|
|
30
|
+
_SEVERITY_ORDER,
|
|
31
|
+
_WRITE_TOOLS,
|
|
32
|
+
LowCacheHitRule,
|
|
33
|
+
OversizedToolResultRule,
|
|
34
|
+
RepeatedFileReadRule,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# Read + write tool names that drive RepeatedFileReadRule's state machine;
|
|
38
|
+
# every other tool_use is a no-op there, so we only fetch these.
|
|
39
|
+
_RW_TOOLS = ("Read", *sorted(_WRITE_TOOLS))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
async def _rows(conn, sql: str, params: tuple = ()) -> list:
|
|
43
|
+
async with conn.execute(sql, params) as cur:
|
|
44
|
+
return await cur.fetchall()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def _build_usage(conn) -> None:
|
|
48
|
+
"""Materialize a small per-message usage temp table in ONE scan, so the
|
|
49
|
+
four usage aggregates (daily / models / prompt_stats / cache rollup)
|
|
50
|
+
don't each re-scan the 145k-row messages table. The parent role/meta
|
|
51
|
+
flags that ``daily`` and ``prompt_stats`` need are resolved here via a
|
|
52
|
+
single PK self-join instead of one join per aggregate.
|
|
53
|
+
"""
|
|
54
|
+
await conn.execute("PRAGMA temp_store=MEMORY")
|
|
55
|
+
await conn.execute("DROP TABLE IF EXISTS temp._usage")
|
|
56
|
+
await conn.execute(
|
|
57
|
+
"""
|
|
58
|
+
CREATE TEMP TABLE _usage AS
|
|
59
|
+
SELECT date(m.timestamp) AS day, m.session_id AS session_id, m.role AS role,
|
|
60
|
+
m.is_sidechain AS is_sidechain, m.is_meta AS is_meta,
|
|
61
|
+
CASE WHEN m.usage_json IS NOT NULL THEN 1 ELSE 0 END AS has_usage,
|
|
62
|
+
m.model AS model,
|
|
63
|
+
COALESCE(m.input_tokens,0) AS it, COALESCE(m.output_tokens,0) AS ot,
|
|
64
|
+
COALESCE(m.cache_creation_tokens,0) AS cc,
|
|
65
|
+
COALESCE(m.cache_read_tokens,0) AS cr,
|
|
66
|
+
CASE WHEN p.id IS NOT NULL AND p.role='user' THEN 1 ELSE 0 END
|
|
67
|
+
AS parent_is_user,
|
|
68
|
+
CASE WHEN p.id IS NOT NULL AND p.is_meta=1 THEN 1 ELSE 0 END
|
|
69
|
+
AS parent_is_meta
|
|
70
|
+
FROM messages m LEFT JOIN messages p ON m.parent_id = p.id
|
|
71
|
+
"""
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
async def _drop_usage(conn) -> None:
|
|
76
|
+
await conn.execute("DROP TABLE IF EXISTS temp._usage")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def daily(conn) -> list[dict[str, Any]]:
|
|
80
|
+
"""Per-UTC-day token totals + prompt/session counts (from ``_usage``).
|
|
81
|
+
Mirrors ``DailyAggregator``: only non-zero-usage messages contribute;
|
|
82
|
+
``prompt_count`` counts non-sidechain/non-meta assistant messages whose
|
|
83
|
+
parent is a user message; ``session_count`` is the distinct sessions."""
|
|
84
|
+
sql = """
|
|
85
|
+
SELECT day,
|
|
86
|
+
SUM(it) AS it, SUM(ot) AS ot, SUM(cc) AS cc, SUM(cr) AS cr,
|
|
87
|
+
COUNT(DISTINCT session_id) AS sessions,
|
|
88
|
+
SUM(CASE WHEN role='assistant' AND is_sidechain=0 AND is_meta=0
|
|
89
|
+
AND parent_is_user=1 THEN 1 ELSE 0 END) AS prompts
|
|
90
|
+
FROM _usage
|
|
91
|
+
WHERE day IS NOT NULL AND (it+ot+cc+cr) > 0
|
|
92
|
+
GROUP BY day ORDER BY day ASC
|
|
93
|
+
"""
|
|
94
|
+
out = []
|
|
95
|
+
for r in await _rows(conn, sql):
|
|
96
|
+
it, ot, cc, cr = r["it"], r["ot"], r["cc"], r["cr"]
|
|
97
|
+
out.append(
|
|
98
|
+
{
|
|
99
|
+
"day": r["day"],
|
|
100
|
+
"input_tokens": it,
|
|
101
|
+
"output_tokens": ot,
|
|
102
|
+
"cache_creation_tokens": cc,
|
|
103
|
+
"cache_read_tokens": cr,
|
|
104
|
+
"total": it + ot + cc + cr,
|
|
105
|
+
"prompt_count": r["prompts"],
|
|
106
|
+
"session_count": r["sessions"],
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
async def models(conn) -> list[dict[str, Any]]:
|
|
113
|
+
"""Token usage per model (from ``_usage``). Mirrors ``ModelUsageScanner``:
|
|
114
|
+
assistant messages with a model and a usage block, sorted by total desc."""
|
|
115
|
+
sql = """
|
|
116
|
+
SELECT model, COUNT(*) AS message_count,
|
|
117
|
+
SUM(it) AS input_tokens, SUM(ot) AS output_tokens,
|
|
118
|
+
SUM(it+ot+cc+cr) AS total_tokens
|
|
119
|
+
FROM _usage
|
|
120
|
+
WHERE role='assistant' AND model IS NOT NULL AND model != '' AND has_usage=1
|
|
121
|
+
GROUP BY model ORDER BY total_tokens DESC, model ASC
|
|
122
|
+
"""
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
"model": r["model"],
|
|
126
|
+
"message_count": r["message_count"],
|
|
127
|
+
"input_tokens": r["input_tokens"],
|
|
128
|
+
"output_tokens": r["output_tokens"],
|
|
129
|
+
"total_tokens": r["total_tokens"],
|
|
130
|
+
}
|
|
131
|
+
for r in await _rows(conn, sql)
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
async def prompt_stats(conn) -> dict[str, Any]:
|
|
136
|
+
"""Distribution of per-prompt token counts (from ``_usage``). Mirrors
|
|
137
|
+
``PromptStatsScanner`` + ``PromptIterator``: one prompt per non-sidechain
|
|
138
|
+
assistant message whose parent is a non-meta user message; percentiles
|
|
139
|
+
reuse the canonical ``compute_stats``."""
|
|
140
|
+
sql = """
|
|
141
|
+
SELECT it, ot, (it+ot+cc+cr) AS tot FROM _usage
|
|
142
|
+
WHERE role='assistant' AND is_sidechain=0
|
|
143
|
+
AND parent_is_user=1 AND parent_is_meta=0
|
|
144
|
+
"""
|
|
145
|
+
inputs, outputs, totals = [], [], []
|
|
146
|
+
for r in await _rows(conn, sql):
|
|
147
|
+
inputs.append(r["it"])
|
|
148
|
+
outputs.append(r["ot"])
|
|
149
|
+
totals.append(r["tot"])
|
|
150
|
+
return {
|
|
151
|
+
"prompt_count": len(inputs),
|
|
152
|
+
"input_tokens": compute_stats(inputs),
|
|
153
|
+
"output_tokens": compute_stats(outputs),
|
|
154
|
+
"total_tokens": compute_stats(totals),
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
async def _low_cache_totals(conn) -> tuple[int, int]:
|
|
159
|
+
"""(total_input, total_cache_read) over assistant messages with usage
|
|
160
|
+
(from ``_usage``) — the inputs LowCacheHitRule needs."""
|
|
161
|
+
row = (
|
|
162
|
+
await _rows(
|
|
163
|
+
conn,
|
|
164
|
+
"SELECT SUM(it+cr+cc) AS total_input, SUM(cr) AS cache_read "
|
|
165
|
+
"FROM _usage WHERE role='assistant' AND has_usage=1",
|
|
166
|
+
)
|
|
167
|
+
)[0]
|
|
168
|
+
return row["total_input"] or 0, row["cache_read"] or 0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _fake_msg(blocks: list) -> Message:
|
|
172
|
+
"""Minimal Message carrying just content blocks, so the real tip rules
|
|
173
|
+
run unchanged on materialized data (guarantees byte-for-byte parity)."""
|
|
174
|
+
return Message(id="x", session_id="x", role="assistant", content_blocks=blocks)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
async def _tips(conn, cache_totals: tuple[int, int]) -> list[dict[str, Any]]:
|
|
178
|
+
"""Reproduce ``TipRegistry`` by running the real rule classes on minimal
|
|
179
|
+
messages rebuilt from ``message_blocks`` + the cache rollup, so
|
|
180
|
+
heuristics, copy, and ordering stay identical to the scanners.
|
|
181
|
+
|
|
182
|
+
``cache_totals`` is ``(total_input, total_cache_read)`` from the usage
|
|
183
|
+
pass, reused here so LowCacheHit needs no extra scan.
|
|
184
|
+
"""
|
|
185
|
+
placeholders = ",".join("?" * len(_RW_TOOLS))
|
|
186
|
+
rw_rows = await _rows(
|
|
187
|
+
conn,
|
|
188
|
+
f"""SELECT mb.tool_name AS name, mb.file_path AS path
|
|
189
|
+
FROM message_blocks mb JOIN messages m ON mb.message_id = m.id
|
|
190
|
+
WHERE mb.btype='tool_use' AND mb.tool_name IN ({placeholders})
|
|
191
|
+
ORDER BY m.timestamp, m.rowid, mb.block_index""",
|
|
192
|
+
_RW_TOOLS,
|
|
193
|
+
)
|
|
194
|
+
repeated = RepeatedFileReadRule().check(
|
|
195
|
+
[
|
|
196
|
+
_fake_msg(
|
|
197
|
+
[
|
|
198
|
+
ToolUseBlock(
|
|
199
|
+
id="t", name=r["name"], input={"file_path": r["path"] or ""}
|
|
200
|
+
)
|
|
201
|
+
]
|
|
202
|
+
)
|
|
203
|
+
for r in rw_rows
|
|
204
|
+
]
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
over_rows = await _rows(
|
|
208
|
+
conn,
|
|
209
|
+
"""SELECT mb.tool_use_id AS tid, mb.result_chars AS chars
|
|
210
|
+
FROM message_blocks mb JOIN messages m ON mb.message_id = m.id
|
|
211
|
+
WHERE mb.btype='tool_result' AND mb.result_chars >= ?
|
|
212
|
+
ORDER BY m.timestamp, m.rowid, mb.block_index""",
|
|
213
|
+
(OversizedToolResultRule()._threshold,),
|
|
214
|
+
)
|
|
215
|
+
oversized = OversizedToolResultRule().check(
|
|
216
|
+
[
|
|
217
|
+
_fake_msg(
|
|
218
|
+
[ToolResultBlock(tool_use_id=r["tid"] or "", content="x" * r["chars"])]
|
|
219
|
+
)
|
|
220
|
+
for r in over_rows
|
|
221
|
+
]
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
total_input, cache_read = cache_totals
|
|
225
|
+
usage_msg = _fake_msg([])
|
|
226
|
+
usage_msg.usage = TokenUsage(
|
|
227
|
+
input_tokens=total_input - cache_read, cache_read_input_tokens=cache_read
|
|
228
|
+
)
|
|
229
|
+
low_cache = LowCacheHitRule().check([usage_msg])
|
|
230
|
+
|
|
231
|
+
tips = sorted(
|
|
232
|
+
repeated + oversized + low_cache,
|
|
233
|
+
key=lambda t: _SEVERITY_ORDER.get(t.severity, 99),
|
|
234
|
+
)
|
|
235
|
+
return [
|
|
236
|
+
{
|
|
237
|
+
"rule_id": t.rule_id,
|
|
238
|
+
"severity": t.severity,
|
|
239
|
+
"title": t.title,
|
|
240
|
+
"detail": t.detail,
|
|
241
|
+
"evidence": t.evidence,
|
|
242
|
+
}
|
|
243
|
+
for t in tips
|
|
244
|
+
]
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
async def compute_bundle(conn, files_top_n: int = 20) -> dict[str, Any]:
|
|
248
|
+
"""The whole-corpus analytics bundle, entirely from materialized data.
|
|
249
|
+
|
|
250
|
+
No ``content_json`` is read: token metrics come from per-message columns,
|
|
251
|
+
tool/file/tip facts from ``message_blocks``. Verified against the Python
|
|
252
|
+
scanners in ``tests/analytics/test_fast_parity.py``.
|
|
253
|
+
|
|
254
|
+
Counts match the scanners exactly; the order *among equal counts* uses a
|
|
255
|
+
deterministic name tiebreak rather than the scanners' first-seen order
|
|
256
|
+
(matching it needs a join + per-row composite key that cost ~600ms for a
|
|
257
|
+
purely cosmetic chart ordering).
|
|
258
|
+
"""
|
|
259
|
+
# One scan into a per-message temp table feeds all four usage aggregates.
|
|
260
|
+
await _build_usage(conn)
|
|
261
|
+
try:
|
|
262
|
+
daily_r = await daily(conn)
|
|
263
|
+
models_r = await models(conn)
|
|
264
|
+
stats_r = await prompt_stats(conn)
|
|
265
|
+
cache_totals = await _low_cache_totals(conn)
|
|
266
|
+
finally:
|
|
267
|
+
await _drop_usage(conn)
|
|
268
|
+
|
|
269
|
+
tool_rows = await _rows(
|
|
270
|
+
conn,
|
|
271
|
+
"""SELECT tool_name AS name, COUNT(*) AS n FROM message_blocks
|
|
272
|
+
WHERE btype='tool_use' AND tool_name IS NOT NULL
|
|
273
|
+
GROUP BY tool_name ORDER BY n DESC, tool_name ASC""",
|
|
274
|
+
)
|
|
275
|
+
file_rows = await _rows(
|
|
276
|
+
conn,
|
|
277
|
+
"""SELECT file_path AS path, COUNT(*) AS n FROM message_blocks
|
|
278
|
+
WHERE btype='tool_use' AND file_path IS NOT NULL AND file_path != ''
|
|
279
|
+
GROUP BY file_path ORDER BY n DESC, file_path ASC LIMIT ?""",
|
|
280
|
+
(files_top_n,),
|
|
281
|
+
)
|
|
282
|
+
return {
|
|
283
|
+
"daily": daily_r,
|
|
284
|
+
"models": models_r,
|
|
285
|
+
"stats": stats_r,
|
|
286
|
+
"tools": [{"tool": r["name"], "count": r["n"]} for r in tool_rows],
|
|
287
|
+
"files": [{"file": r["path"], "count": r["n"]} for r in file_rows],
|
|
288
|
+
"tips": await _tips(conn, cache_totals),
|
|
289
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from datetime import date
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class TokenBreakdown:
|
|
8
|
+
input_tokens: int = 0
|
|
9
|
+
output_tokens: int = 0
|
|
10
|
+
cache_creation_tokens: int = 0
|
|
11
|
+
cache_read_tokens: int = 0
|
|
12
|
+
|
|
13
|
+
@property
|
|
14
|
+
def total(self) -> int:
|
|
15
|
+
return (
|
|
16
|
+
self.input_tokens
|
|
17
|
+
+ self.output_tokens
|
|
18
|
+
+ self.cache_creation_tokens
|
|
19
|
+
+ self.cache_read_tokens
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def __add__(self, other: "TokenBreakdown") -> "TokenBreakdown":
|
|
23
|
+
return TokenBreakdown(
|
|
24
|
+
input_tokens=self.input_tokens + other.input_tokens,
|
|
25
|
+
output_tokens=self.output_tokens + other.output_tokens,
|
|
26
|
+
cache_creation_tokens=self.cache_creation_tokens
|
|
27
|
+
+ other.cache_creation_tokens,
|
|
28
|
+
cache_read_tokens=self.cache_read_tokens + other.cache_read_tokens,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class PromptCost:
|
|
34
|
+
"""Token breakdown for a single user→assistant turn."""
|
|
35
|
+
|
|
36
|
+
user_message_id: str
|
|
37
|
+
assistant_message_id: str
|
|
38
|
+
model: str = "unknown"
|
|
39
|
+
breakdown: TokenBreakdown = field(default_factory=TokenBreakdown)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class DailyBucket:
|
|
44
|
+
"""Aggregated token breakdown for a UTC calendar day."""
|
|
45
|
+
|
|
46
|
+
day: date
|
|
47
|
+
breakdown: TokenBreakdown = field(default_factory=TokenBreakdown)
|
|
48
|
+
session_count: int = 0
|
|
49
|
+
prompt_count: int = 0
|