agy-token-stats 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agy_token_stats-0.1.0/LICENSE +21 -0
- agy_token_stats-0.1.0/PKG-INFO +199 -0
- agy_token_stats-0.1.0/README.md +167 -0
- agy_token_stats-0.1.0/pyproject.toml +53 -0
- agy_token_stats-0.1.0/setup.cfg +4 -0
- agy_token_stats-0.1.0/src/agy_token_stats/__init__.py +96 -0
- agy_token_stats-0.1.0/src/agy_token_stats/__main__.py +8 -0
- agy_token_stats-0.1.0/src/agy_token_stats/aggregator.py +412 -0
- agy_token_stats-0.1.0/src/agy_token_stats/cli.py +263 -0
- agy_token_stats-0.1.0/src/agy_token_stats/discovery.py +378 -0
- agy_token_stats-0.1.0/src/agy_token_stats/formatters.py +480 -0
- agy_token_stats-0.1.0/src/agy_token_stats/models.py +402 -0
- agy_token_stats-0.1.0/src/agy_token_stats/parser.py +378 -0
- agy_token_stats-0.1.0/src/agy_token_stats/time_window.py +180 -0
- agy_token_stats-0.1.0/src/agy_token_stats.egg-info/PKG-INFO +199 -0
- agy_token_stats-0.1.0/src/agy_token_stats.egg-info/SOURCES.txt +21 -0
- agy_token_stats-0.1.0/src/agy_token_stats.egg-info/dependency_links.txt +1 -0
- agy_token_stats-0.1.0/src/agy_token_stats.egg-info/entry_points.txt +2 -0
- agy_token_stats-0.1.0/src/agy_token_stats.egg-info/requires.txt +6 -0
- agy_token_stats-0.1.0/src/agy_token_stats.egg-info/top_level.txt +1 -0
- agy_token_stats-0.1.0/tests/test_aggregator_and_cli.py +673 -0
- agy_token_stats-0.1.0/tests/test_parser_and_discovery.py +784 -0
- agy_token_stats-0.1.0/tests/test_time_window.py +168 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 vikrant82
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agy-token-stats
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Estimate and report token usage statistics across Antigravity (agy) CLI and IDE sessions, including subagent trees.
|
|
5
|
+
Author-email: vikrant82 <vikrant82@users.noreply.github.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/vikrant82/agy-token-stats
|
|
8
|
+
Project-URL: Repository, https://github.com/vikrant82/agy-token-stats
|
|
9
|
+
Project-URL: Issues, https://github.com/vikrant82/agy-token-stats/issues
|
|
10
|
+
Keywords: antigravity,agy,tokens,analytics,cli,llm
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
22
|
+
Classifier: Topic :: Utilities
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
30
|
+
Requires-Dist: mypy>=1.5.0; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# agy-token-stats
|
|
34
|
+
|
|
35
|
+
Zero-dependency CLI and Python library for analyzing token usage, prompt cache efficiency, thinking token decomposition, model breakdowns, context growth, and subagent trees across **Antigravity (`agy`)** CLI and IDE sessions.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## Overview
|
|
40
|
+
|
|
41
|
+
Antigravity records structured per-step execution transcripts for every root session and spawned subagent conversation across two local brain directories:
|
|
42
|
+
|
|
43
|
+
- **CLI brain**: `~/.gemini/antigravity-cli/brain/<conversation-id>/`
|
|
44
|
+
- **IDE brain**: `~/.gemini/antigravity/brain/<conversation-id>/`
|
|
45
|
+
|
|
46
|
+
Within each conversation directory:
|
|
47
|
+
|
|
48
|
+
- `.system_generated/logs/transcript.jsonl` stores chronological step records (`USER_INPUT`, `PLANNER_RESPONSE`, `SYSTEM_MESSAGE`, etc.), including native token counters (`input_tokens`, `cache_read_tokens`, `output_tokens`), `thinking` text, `content`, and tool call payloads.
|
|
49
|
+
- `.system_generated/subagents/<child-id>.json` stores explicit parent-to-child subagent descriptors (`typeName`, `role`, `spawnStepIndex`, `workspaceUris`).
|
|
50
|
+
- `~/.gemini/antigravity-cli/history.jsonl` (optional) stores workspace paths and user prompt history for CLI sessions.
|
|
51
|
+
|
|
52
|
+
`agy-token-stats` discovers all sessions across CLI and IDE brains, reconstructs multi-level subagent trees (`root` -> `implementer` / `oracle` / `reviewer` / `bash-executor`), tracks and propagates model selections across steps and subagents, filters by local-time windows or `--model`, and produces human-readable tables or structured JSON reports without double counting.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Token Metrics, Thinking Decomposition & Subagent Tree Rollups
|
|
57
|
+
|
|
58
|
+
### Token Metric Definitions
|
|
59
|
+
|
|
60
|
+
| Metric | Formula / Meaning |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `uncached_input` | Sum of native `input_tokens` across `PLANNER_RESPONSE` steps (prompt tokens not served from cache). |
|
|
63
|
+
| `cache_read` | Sum of native `cache_read_tokens` across `PLANNER_RESPONSE` steps. |
|
|
64
|
+
| `gross_input` | `uncached_input + cache_read` — total prompt context processed by the model across all calls. |
|
|
65
|
+
| `cache_hit_pct` | `cache_read / gross_input * 100` (`0.0%` when `gross_input == 0`). |
|
|
66
|
+
| `output_tokens` | Sum of native `output_tokens` generated across `PLANNER_RESPONSE` steps (includes thinking + visible content + tool calls). |
|
|
67
|
+
| `thinking_tokens` | Estimated portion of `output_tokens` spent on `thinking` text (`0 <= thinking_tokens <= output_tokens`). |
|
|
68
|
+
| `response_tokens` | `max(0, output_tokens - thinking_tokens)` — estimated non-thinking output tokens (visible content + tool calls). |
|
|
69
|
+
| `thinking_pct` | `thinking_tokens / output_tokens * 100` (`0.0%` when `output_tokens == 0`). |
|
|
70
|
+
| `thinking_steps` | Number of `PLANNER_RESPONSE` steps with non-empty `thinking` text. |
|
|
71
|
+
| `net_tokens` | `uncached_input + output_tokens` — non-cached tokens processed plus output generated. (`thinking_tokens` is a decomposition of `output_tokens` and is never double-counted.) |
|
|
72
|
+
| `total_tokens` | `gross_input + output_tokens` — total tokens processed including prompt cache reads. |
|
|
73
|
+
| `initial_context` | `gross_input` of the first `PLANNER_RESPONSE` step in the session/window. |
|
|
74
|
+
| `peak_context` | Maximum single-step context (`input_tokens + cache_read_tokens + output_tokens`) across included `PLANNER_RESPONSE` steps. |
|
|
75
|
+
|
|
76
|
+
### Thinking Token Estimation
|
|
77
|
+
|
|
78
|
+
Native Antigravity `PLANNER_RESPONSE` entries emit a single combined `output_tokens` counter covering internal reasoning (`thinking`), visible response text (`content`), and serialized tool invocations (`tool_calls`).
|
|
79
|
+
|
|
80
|
+
For each `PLANNER_RESPONSE` step with non-empty `thinking`:
|
|
81
|
+
1. Untruncated text sizes (`thinking_len`, `content_len`, `tool_calls_len`) are computed, expanding any `<truncated N bytes>` markers recorded in `transcript.jsonl`.
|
|
82
|
+
2. When `output_tokens > 0`:
|
|
83
|
+
- If `content_len + tool_calls_len == 0`, `thinking_tokens = output_tokens`.
|
|
84
|
+
- Otherwise, `thinking_tokens = min(output_tokens, max(1, round(output_tokens * (thinking_len / (thinking_len + content_len + tool_calls_len)))))`.
|
|
85
|
+
3. `response_tokens = output_tokens - thinking_tokens`, preserving the exact invariant `thinking_tokens + response_tokens == output_tokens`.
|
|
86
|
+
|
|
87
|
+
> **Fallback character estimation**: If a legacy or truncated `PLANNER_RESPONSE` step omits both `input_tokens` and `output_tokens`, `agy-token-stats` estimates `input_tokens = ceil(accumulated_context_chars / 4)` and `output_tokens = ceil(step_text_chars / 4)`, sets `is_estimated = True`, and reports the exact count of estimated steps in the header.
|
|
88
|
+
|
|
89
|
+
### Model Attribution & Subagent Inheritance
|
|
90
|
+
|
|
91
|
+
- **Sequential step tracking**: `<USER_SETTINGS_CHANGE>` model tags (`changed setting Model Selection from ... to <Model>.`) are tracked chronologically across `USER_INPUT` steps. If a user switches models mid-session, steps before and after the switch are attributed to their respective models in `by_model`.
|
|
92
|
+
- **Subagent inheritance**: Subagent transcripts begin with `SYSTEM_MESSAGE` (`step_index: 0`) and omit `<USER_SETTINGS_CHANGE>`. During discovery, models are propagated top-down from parent sessions to child and grandchild subagents using the parent's active model at `spawnStepIndex` (or any explicit `invoke_subagent` `Model` override).
|
|
93
|
+
|
|
94
|
+
### Subagent Tree Rollup vs. Flat Mode
|
|
95
|
+
|
|
96
|
+
1. **Parent-Child Linking**:
|
|
97
|
+
- **Primary**: Explicit descriptor files in `<parent>/.system_generated/subagents/<child>.json` provide `parent_id`, `typeName` (`oracle`, `implementer`, `reviewer`, `bash-executor`, etc.), `role`, `spawnStepIndex`, and workspace fallback.
|
|
98
|
+
- **Fallback**: If a descriptor file is missing, an initial `SYSTEM_MESSAGE` step containing `sender=<parent-uuid>` links the subagent to its parent session (with cycle prevention).
|
|
99
|
+
2. **No Double Counting**:
|
|
100
|
+
- `overall`, `by_source`, `by_role`, `by_model`, and `daily` aggregate each active step exactly once.
|
|
101
|
+
- In **tree mode** (default), the `Sessions` table displays top-level root sessions with `tree_metrics` (parent `self_metrics` + all descendant subagents' `self_metrics`). Passing `--show-subagents` (or `--session <id>`) expands an indented breakdown showing `├─ <id> (self)` and each active child/grandchild subagent row.
|
|
102
|
+
- In **flat mode** (`--flat`), every conversation UUID is listed independently with `tree_metrics == self_metrics`.
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
## Installation
|
|
107
|
+
|
|
108
|
+
`agy-token-stats` requires **Python 3.9+** and has **zero third-party runtime dependencies**.
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
# Install from PyPI
|
|
112
|
+
pip install agy-token-stats
|
|
113
|
+
# or isolated CLI via uv / pipx
|
|
114
|
+
uv tool install agy-token-stats
|
|
115
|
+
pipx install agy-token-stats
|
|
116
|
+
|
|
117
|
+
# Or install directly from GitHub
|
|
118
|
+
uv tool install git+https://github.com/vikrant82/agy-token-stats.git
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## CLI Options
|
|
124
|
+
|
|
125
|
+
| Flag | Description |
|
|
126
|
+
|---|---|
|
|
127
|
+
| `--source {all,cli,ide}` | Brain source(s) to scan when `--brain-dir` is not specified (default: `all`). |
|
|
128
|
+
| `--brain-dir [LABEL=]PATH` | Override brain root directory to scan (repeatable, e.g. `--brain-dir cli=/path/to/brain`). |
|
|
129
|
+
| `--history-file PATH` | Path to CLI `history.jsonl` for workspace/title enrichment. |
|
|
130
|
+
| `--today` | Include activity from today `00:00:00` local time to now. |
|
|
131
|
+
| `--yesterday` | Include activity from yesterday `00:00:00` to `23:59:59.999999` local time. |
|
|
132
|
+
| `--last-n-days N`, `--days N` | Include activity from the last `N` calendar days in local time including today (`N=1` equals `--today`; `N=7` covers today + 6 preceding days). |
|
|
133
|
+
| `--last-n-hours H`, `--hours H` | Include activity from the rolling last `H` hours up to now. |
|
|
134
|
+
| `--since DATE_OR_ISO` | Lower bound in local time (`YYYY-MM-DD` at `00:00:00` or ISO-8601 timestamp). |
|
|
135
|
+
| `--until DATE_OR_ISO` | Upper bound in local time (`YYYY-MM-DD` at `23:59:59.999999` or ISO-8601 timestamp). |
|
|
136
|
+
| `--by-session` | Filter whole sessions by session start time instead of individual step timestamps. |
|
|
137
|
+
| `--session ID_OR_PREFIX` | Filter to a conversation UUID (or prefix) and expand its subagent tree. |
|
|
138
|
+
| `--model PATTERN` | Filter steps/sessions by model name using a case-insensitive substring match (e.g. `--model claude`, `--model flash`). |
|
|
139
|
+
| `--flat` | Treat every conversation UUID independently without rolling subagents into parent roots. |
|
|
140
|
+
| `--show-subagents` | Display indented child subagent rows beneath each parent session in the sessions table. |
|
|
141
|
+
| `--top N` | Number of sessions to show in the sessions table (`0` for all; default: `20`). |
|
|
142
|
+
| `--daily` / `--no-daily` | Force-enable or hide the local-date daily breakdown table. |
|
|
143
|
+
| `--json` | Emit structured JSON output instead of the human-readable report. |
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## Usage Examples
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
# Today's token usage across both CLI and IDE brains
|
|
151
|
+
agy-token-stats --today
|
|
152
|
+
|
|
153
|
+
# Filter to a specific model family over the last 7 calendar days
|
|
154
|
+
agy-token-stats --days 7 --model claude
|
|
155
|
+
|
|
156
|
+
# Last 7 calendar days with daily breakdown and expanded subagent trees
|
|
157
|
+
agy-token-stats --days 7 --daily --show-subagents
|
|
158
|
+
|
|
159
|
+
# Rolling last 4 hours for IDE sessions only
|
|
160
|
+
agy-token-stats --source ide --hours 4
|
|
161
|
+
|
|
162
|
+
# Drill into a specific root or subagent session by UUID prefix
|
|
163
|
+
agy-token-stats --session 179c4f24
|
|
164
|
+
|
|
165
|
+
# Export JSON report for a custom date range
|
|
166
|
+
agy-token-stats --since 2026-04-01 --until 2026-04-19 --json
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## Package Architecture
|
|
172
|
+
|
|
173
|
+
```text
|
|
174
|
+
agy-token-stats/
|
|
175
|
+
├── LICENSE
|
|
176
|
+
├── pyproject.toml
|
|
177
|
+
├── README.md
|
|
178
|
+
├── src/agy_token_stats/
|
|
179
|
+
│ ├── __init__.py # Public package API and version metadata
|
|
180
|
+
│ ├── __main__.py # `python3 -m agy_token_stats` entry point
|
|
181
|
+
│ ├── models.py # Dataclasses: StepRecord, SessionRecord, TokenMetrics, StatsReport, etc.
|
|
182
|
+
│ ├── time_window.py # Timezone resolution, ISO-8601 parsing, calendar/rolling window bounds
|
|
183
|
+
│ ├── parser.py # Transcript JSONL step parsing, thinking split, model tracking, char/4 fallback
|
|
184
|
+
│ ├── discovery.py # Brain directory scanning, history.jsonl enrichment, subagent model inheritance
|
|
185
|
+
│ ├── aggregator.py # Step/session/model filtering and tree/flat rollup aggregation
|
|
186
|
+
│ ├── formatters.py # Aligned human-readable CLI tables and JSON report serialization
|
|
187
|
+
│ └── cli.py # Argument parser, source path resolution, and CLI main()
|
|
188
|
+
└── tests/
|
|
189
|
+
├── helpers.py
|
|
190
|
+
├── test_time_window.py
|
|
191
|
+
├── test_parser_and_discovery.py
|
|
192
|
+
└── test_aggregator_and_cli.py
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Running Tests
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
python3 -m unittest discover -s tests -t . -v
|
|
199
|
+
```
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# agy-token-stats
|
|
2
|
+
|
|
3
|
+
Zero-dependency CLI and Python library for analyzing token usage, prompt cache efficiency, thinking token decomposition, model breakdowns, context growth, and subagent trees across **Antigravity (`agy`)** CLI and IDE sessions.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Overview
|
|
8
|
+
|
|
9
|
+
Antigravity records structured per-step execution transcripts for every root session and spawned subagent conversation across two local brain directories:
|
|
10
|
+
|
|
11
|
+
- **CLI brain**: `~/.gemini/antigravity-cli/brain/<conversation-id>/`
|
|
12
|
+
- **IDE brain**: `~/.gemini/antigravity/brain/<conversation-id>/`
|
|
13
|
+
|
|
14
|
+
Within each conversation directory:
|
|
15
|
+
|
|
16
|
+
- `.system_generated/logs/transcript.jsonl` stores chronological step records (`USER_INPUT`, `PLANNER_RESPONSE`, `SYSTEM_MESSAGE`, etc.), including native token counters (`input_tokens`, `cache_read_tokens`, `output_tokens`), `thinking` text, `content`, and tool call payloads.
|
|
17
|
+
- `.system_generated/subagents/<child-id>.json` stores explicit parent-to-child subagent descriptors (`typeName`, `role`, `spawnStepIndex`, `workspaceUris`).
|
|
18
|
+
- `~/.gemini/antigravity-cli/history.jsonl` (optional) stores workspace paths and user prompt history for CLI sessions.
|
|
19
|
+
|
|
20
|
+
`agy-token-stats` discovers all sessions across CLI and IDE brains, reconstructs multi-level subagent trees (`root` -> `implementer` / `oracle` / `reviewer` / `bash-executor`), tracks and propagates model selections across steps and subagents, filters by local-time windows or `--model`, and produces human-readable tables or structured JSON reports without double counting.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Token Metrics, Thinking Decomposition & Subagent Tree Rollups
|
|
25
|
+
|
|
26
|
+
### Token Metric Definitions
|
|
27
|
+
|
|
28
|
+
| Metric | Formula / Meaning |
|
|
29
|
+
|---|---|
|
|
30
|
+
| `uncached_input` | Sum of native `input_tokens` across `PLANNER_RESPONSE` steps (prompt tokens not served from cache). |
|
|
31
|
+
| `cache_read` | Sum of native `cache_read_tokens` across `PLANNER_RESPONSE` steps. |
|
|
32
|
+
| `gross_input` | `uncached_input + cache_read` — total prompt context processed by the model across all calls. |
|
|
33
|
+
| `cache_hit_pct` | `cache_read / gross_input * 100` (`0.0%` when `gross_input == 0`). |
|
|
34
|
+
| `output_tokens` | Sum of native `output_tokens` generated across `PLANNER_RESPONSE` steps (includes thinking + visible content + tool calls). |
|
|
35
|
+
| `thinking_tokens` | Estimated portion of `output_tokens` spent on `thinking` text (`0 <= thinking_tokens <= output_tokens`). |
|
|
36
|
+
| `response_tokens` | `max(0, output_tokens - thinking_tokens)` — estimated non-thinking output tokens (visible content + tool calls). |
|
|
37
|
+
| `thinking_pct` | `thinking_tokens / output_tokens * 100` (`0.0%` when `output_tokens == 0`). |
|
|
38
|
+
| `thinking_steps` | Number of `PLANNER_RESPONSE` steps with non-empty `thinking` text. |
|
|
39
|
+
| `net_tokens` | `uncached_input + output_tokens` — non-cached tokens processed plus output generated. (`thinking_tokens` is a decomposition of `output_tokens` and is never double-counted.) |
|
|
40
|
+
| `total_tokens` | `gross_input + output_tokens` — total tokens processed including prompt cache reads. |
|
|
41
|
+
| `initial_context` | `gross_input` of the first `PLANNER_RESPONSE` step in the session/window. |
|
|
42
|
+
| `peak_context` | Maximum single-step context (`input_tokens + cache_read_tokens + output_tokens`) across included `PLANNER_RESPONSE` steps. |
|
|
43
|
+
|
|
44
|
+
### Thinking Token Estimation
|
|
45
|
+
|
|
46
|
+
Native Antigravity `PLANNER_RESPONSE` entries emit a single combined `output_tokens` counter covering internal reasoning (`thinking`), visible response text (`content`), and serialized tool invocations (`tool_calls`).
|
|
47
|
+
|
|
48
|
+
For each `PLANNER_RESPONSE` step with non-empty `thinking`:
|
|
49
|
+
1. Untruncated text sizes (`thinking_len`, `content_len`, `tool_calls_len`) are computed, expanding any `<truncated N bytes>` markers recorded in `transcript.jsonl`.
|
|
50
|
+
2. When `output_tokens > 0`:
|
|
51
|
+
- If `content_len + tool_calls_len == 0`, `thinking_tokens = output_tokens`.
|
|
52
|
+
- Otherwise, `thinking_tokens = min(output_tokens, max(1, round(output_tokens * (thinking_len / (thinking_len + content_len + tool_calls_len)))))`.
|
|
53
|
+
3. `response_tokens = output_tokens - thinking_tokens`, preserving the exact invariant `thinking_tokens + response_tokens == output_tokens`.
|
|
54
|
+
|
|
55
|
+
> **Fallback character estimation**: If a legacy or truncated `PLANNER_RESPONSE` step omits both `input_tokens` and `output_tokens`, `agy-token-stats` estimates `input_tokens = ceil(accumulated_context_chars / 4)` and `output_tokens = ceil(step_text_chars / 4)`, sets `is_estimated = True`, and reports the exact count of estimated steps in the header.
|
|
56
|
+
|
|
57
|
+
### Model Attribution & Subagent Inheritance
|
|
58
|
+
|
|
59
|
+
- **Sequential step tracking**: `<USER_SETTINGS_CHANGE>` model tags (`changed setting Model Selection from ... to <Model>.`) are tracked chronologically across `USER_INPUT` steps. If a user switches models mid-session, steps before and after the switch are attributed to their respective models in `by_model`.
|
|
60
|
+
- **Subagent inheritance**: Subagent transcripts begin with `SYSTEM_MESSAGE` (`step_index: 0`) and omit `<USER_SETTINGS_CHANGE>`. During discovery, models are propagated top-down from parent sessions to child and grandchild subagents using the parent's active model at `spawnStepIndex` (or any explicit `invoke_subagent` `Model` override).
|
|
61
|
+
|
|
62
|
+
### Subagent Tree Rollup vs. Flat Mode
|
|
63
|
+
|
|
64
|
+
1. **Parent-Child Linking**:
|
|
65
|
+
- **Primary**: Explicit descriptor files in `<parent>/.system_generated/subagents/<child>.json` provide `parent_id`, `typeName` (`oracle`, `implementer`, `reviewer`, `bash-executor`, etc.), `role`, `spawnStepIndex`, and workspace fallback.
|
|
66
|
+
- **Fallback**: If a descriptor file is missing, an initial `SYSTEM_MESSAGE` step containing `sender=<parent-uuid>` links the subagent to its parent session (with cycle prevention).
|
|
67
|
+
2. **No Double Counting**:
|
|
68
|
+
- `overall`, `by_source`, `by_role`, `by_model`, and `daily` aggregate each active step exactly once.
|
|
69
|
+
- In **tree mode** (default), the `Sessions` table displays top-level root sessions with `tree_metrics` (parent `self_metrics` + all descendant subagents' `self_metrics`). Passing `--show-subagents` (or `--session <id>`) expands an indented breakdown showing `├─ <id> (self)` and each active child/grandchild subagent row.
|
|
70
|
+
- In **flat mode** (`--flat`), every conversation UUID is listed independently with `tree_metrics == self_metrics`.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Installation
|
|
75
|
+
|
|
76
|
+
`agy-token-stats` requires **Python 3.9+** and has **zero third-party runtime dependencies**.
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
# Install from PyPI
|
|
80
|
+
pip install agy-token-stats
|
|
81
|
+
# or isolated CLI via uv / pipx
|
|
82
|
+
uv tool install agy-token-stats
|
|
83
|
+
pipx install agy-token-stats
|
|
84
|
+
|
|
85
|
+
# Or install directly from GitHub
|
|
86
|
+
uv tool install git+https://github.com/vikrant82/agy-token-stats.git
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## CLI Options
|
|
92
|
+
|
|
93
|
+
| Flag | Description |
|
|
94
|
+
|---|---|
|
|
95
|
+
| `--source {all,cli,ide}` | Brain source(s) to scan when `--brain-dir` is not specified (default: `all`). |
|
|
96
|
+
| `--brain-dir [LABEL=]PATH` | Override brain root directory to scan (repeatable, e.g. `--brain-dir cli=/path/to/brain`). |
|
|
97
|
+
| `--history-file PATH` | Path to CLI `history.jsonl` for workspace/title enrichment. |
|
|
98
|
+
| `--today` | Include activity from today `00:00:00` local time to now. |
|
|
99
|
+
| `--yesterday` | Include activity from yesterday `00:00:00` to `23:59:59.999999` local time. |
|
|
100
|
+
| `--last-n-days N`, `--days N` | Include activity from the last `N` calendar days in local time including today (`N=1` equals `--today`; `N=7` covers today + 6 preceding days). |
|
|
101
|
+
| `--last-n-hours H`, `--hours H` | Include activity from the rolling last `H` hours up to now. |
|
|
102
|
+
| `--since DATE_OR_ISO` | Lower bound in local time (`YYYY-MM-DD` at `00:00:00` or ISO-8601 timestamp). |
|
|
103
|
+
| `--until DATE_OR_ISO` | Upper bound in local time (`YYYY-MM-DD` at `23:59:59.999999` or ISO-8601 timestamp). |
|
|
104
|
+
| `--by-session` | Filter whole sessions by session start time instead of individual step timestamps. |
|
|
105
|
+
| `--session ID_OR_PREFIX` | Filter to a conversation UUID (or prefix) and expand its subagent tree. |
|
|
106
|
+
| `--model PATTERN` | Filter steps/sessions by model name using a case-insensitive substring match (e.g. `--model claude`, `--model flash`). |
|
|
107
|
+
| `--flat` | Treat every conversation UUID independently without rolling subagents into parent roots. |
|
|
108
|
+
| `--show-subagents` | Display indented child subagent rows beneath each parent session in the sessions table. |
|
|
109
|
+
| `--top N` | Number of sessions to show in the sessions table (`0` for all; default: `20`). |
|
|
110
|
+
| `--daily` / `--no-daily` | Force-enable or hide the local-date daily breakdown table. |
|
|
111
|
+
| `--json` | Emit structured JSON output instead of the human-readable report. |
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## Usage Examples
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
# Today's token usage across both CLI and IDE brains
|
|
119
|
+
agy-token-stats --today
|
|
120
|
+
|
|
121
|
+
# Filter to a specific model family over the last 7 calendar days
|
|
122
|
+
agy-token-stats --days 7 --model claude
|
|
123
|
+
|
|
124
|
+
# Last 7 calendar days with daily breakdown and expanded subagent trees
|
|
125
|
+
agy-token-stats --days 7 --daily --show-subagents
|
|
126
|
+
|
|
127
|
+
# Rolling last 4 hours for IDE sessions only
|
|
128
|
+
agy-token-stats --source ide --hours 4
|
|
129
|
+
|
|
130
|
+
# Drill into a specific root or subagent session by UUID prefix
|
|
131
|
+
agy-token-stats --session 179c4f24
|
|
132
|
+
|
|
133
|
+
# Export JSON report for a custom date range
|
|
134
|
+
agy-token-stats --since 2026-04-01 --until 2026-04-19 --json
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## Package Architecture
|
|
140
|
+
|
|
141
|
+
```text
|
|
142
|
+
agy-token-stats/
|
|
143
|
+
├── LICENSE
|
|
144
|
+
├── pyproject.toml
|
|
145
|
+
├── README.md
|
|
146
|
+
├── src/agy_token_stats/
|
|
147
|
+
│ ├── __init__.py # Public package API and version metadata
|
|
148
|
+
│ ├── __main__.py # `python3 -m agy_token_stats` entry point
|
|
149
|
+
│ ├── models.py # Dataclasses: StepRecord, SessionRecord, TokenMetrics, StatsReport, etc.
|
|
150
|
+
│ ├── time_window.py # Timezone resolution, ISO-8601 parsing, calendar/rolling window bounds
|
|
151
|
+
│ ├── parser.py # Transcript JSONL step parsing, thinking split, model tracking, char/4 fallback
|
|
152
|
+
│ ├── discovery.py # Brain directory scanning, history.jsonl enrichment, subagent model inheritance
|
|
153
|
+
│ ├── aggregator.py # Step/session/model filtering and tree/flat rollup aggregation
|
|
154
|
+
│ ├── formatters.py # Aligned human-readable CLI tables and JSON report serialization
|
|
155
|
+
│ └── cli.py # Argument parser, source path resolution, and CLI main()
|
|
156
|
+
└── tests/
|
|
157
|
+
├── helpers.py
|
|
158
|
+
├── test_time_window.py
|
|
159
|
+
├── test_parser_and_discovery.py
|
|
160
|
+
└── test_aggregator_and_cli.py
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### Running Tests
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
python3 -m unittest discover -s tests -t . -v
|
|
167
|
+
```
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "agy-token-stats"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Estimate and report token usage statistics across Antigravity (agy) CLI and IDE sessions, including subagent trees."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "vikrant82", email = "vikrant82@users.noreply.github.com" },
|
|
14
|
+
]
|
|
15
|
+
keywords = ["antigravity", "agy", "tokens", "analytics", "cli", "llm"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.9",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
28
|
+
"Topic :: Utilities",
|
|
29
|
+
]
|
|
30
|
+
dependencies = []
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
dev = [
|
|
34
|
+
"pytest>=7.4.0",
|
|
35
|
+
"pytest-cov>=4.1.0",
|
|
36
|
+
"ruff>=0.1.0",
|
|
37
|
+
"mypy>=1.5.0",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/vikrant82/agy-token-stats"
|
|
42
|
+
Repository = "https://github.com/vikrant82/agy-token-stats"
|
|
43
|
+
Issues = "https://github.com/vikrant82/agy-token-stats/issues"
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
agy-token-stats = "agy_token_stats.cli:main"
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.packages.find]
|
|
49
|
+
where = ["src"]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
|
53
|
+
pythonpath = ["src"]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Antigravity (`agy`) token usage statistics and subagent tree analyzer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from agy_token_stats.aggregator import (
|
|
6
|
+
_collect_descendants,
|
|
7
|
+
_find_root_session_id,
|
|
8
|
+
_resolve_root_id,
|
|
9
|
+
analyze_sessions,
|
|
10
|
+
compute_stats_report,
|
|
11
|
+
)
|
|
12
|
+
from agy_token_stats.cli import (
|
|
13
|
+
_resolve_brain_sources,
|
|
14
|
+
build_arg_parser,
|
|
15
|
+
main,
|
|
16
|
+
resolve_brain_sources,
|
|
17
|
+
)
|
|
18
|
+
from agy_token_stats.discovery import (
|
|
19
|
+
DEFAULT_CLI_BRAIN_DIR,
|
|
20
|
+
DEFAULT_CLI_HISTORY_PATH,
|
|
21
|
+
DEFAULT_IDE_BRAIN_DIR,
|
|
22
|
+
SKIPPED_BRAIN_DIRS,
|
|
23
|
+
_load_subagent_descriptors,
|
|
24
|
+
_safe_is_dir,
|
|
25
|
+
discover_sessions,
|
|
26
|
+
infer_source_kind,
|
|
27
|
+
load_cli_history_metadata,
|
|
28
|
+
)
|
|
29
|
+
from agy_token_stats.formatters import (
|
|
30
|
+
format_human_report,
|
|
31
|
+
format_json_report,
|
|
32
|
+
)
|
|
33
|
+
from agy_token_stats.models import (
|
|
34
|
+
DailyStats,
|
|
35
|
+
SessionRecord,
|
|
36
|
+
SessionSummary,
|
|
37
|
+
StatsReport,
|
|
38
|
+
StepRecord,
|
|
39
|
+
SubagentLink,
|
|
40
|
+
TimeWindow,
|
|
41
|
+
TokenMetrics,
|
|
42
|
+
)
|
|
43
|
+
from agy_token_stats.parser import (
|
|
44
|
+
_clean_prompt_preview,
|
|
45
|
+
_safe_is_file,
|
|
46
|
+
_step_text_chars,
|
|
47
|
+
parse_transcript_file,
|
|
48
|
+
)
|
|
49
|
+
from agy_token_stats.time_window import (
|
|
50
|
+
get_default_local_tz,
|
|
51
|
+
parse_bound_datetime,
|
|
52
|
+
parse_iso_timestamp,
|
|
53
|
+
resolve_time_window,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
__version__ = "0.1.0"
|
|
57
|
+
|
|
58
|
+
__all__ = [
|
|
59
|
+
"__version__",
|
|
60
|
+
"DEFAULT_CLI_BRAIN_DIR",
|
|
61
|
+
"DEFAULT_CLI_HISTORY_PATH",
|
|
62
|
+
"DEFAULT_IDE_BRAIN_DIR",
|
|
63
|
+
"SKIPPED_BRAIN_DIRS",
|
|
64
|
+
"DailyStats",
|
|
65
|
+
"SessionRecord",
|
|
66
|
+
"SessionSummary",
|
|
67
|
+
"StatsReport",
|
|
68
|
+
"StepRecord",
|
|
69
|
+
"SubagentLink",
|
|
70
|
+
"TimeWindow",
|
|
71
|
+
"TokenMetrics",
|
|
72
|
+
"_clean_prompt_preview",
|
|
73
|
+
"_collect_descendants",
|
|
74
|
+
"_find_root_session_id",
|
|
75
|
+
"_load_subagent_descriptors",
|
|
76
|
+
"_resolve_brain_sources",
|
|
77
|
+
"_resolve_root_id",
|
|
78
|
+
"_safe_is_dir",
|
|
79
|
+
"_safe_is_file",
|
|
80
|
+
"_step_text_chars",
|
|
81
|
+
"analyze_sessions",
|
|
82
|
+
"build_arg_parser",
|
|
83
|
+
"compute_stats_report",
|
|
84
|
+
"discover_sessions",
|
|
85
|
+
"format_human_report",
|
|
86
|
+
"format_json_report",
|
|
87
|
+
"get_default_local_tz",
|
|
88
|
+
"infer_source_kind",
|
|
89
|
+
"load_cli_history_metadata",
|
|
90
|
+
"main",
|
|
91
|
+
"parse_bound_datetime",
|
|
92
|
+
"parse_iso_timestamp",
|
|
93
|
+
"parse_transcript_file",
|
|
94
|
+
"resolve_brain_sources",
|
|
95
|
+
"resolve_time_window",
|
|
96
|
+
]
|