usage-cli 0.29.32__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- adapters/__init__.py +5 -0
- adapters/agy.py +68 -0
- adapters/claude.py +215 -0
- adapters/codex.py +209 -0
- adapters/rate_limits.py +76 -0
- adapters/registry.py +17 -0
- adapters/types.py +139 -0
- agy_disk_cache.py +135 -0
- agy_loader.py +416 -0
- agy_quota_probe.py +748 -0
- agy_window_keeper.py +185 -0
- analyzer/__init__.py +5 -0
- analyzer/aggregator.py +139 -0
- analyzer/blocks.py +80 -0
- analyzer/diagnoser.py +638 -0
- analyzer/insights.py +277 -0
- analyzer/persona_loader.py +199 -0
- analyzer/reporter.py +989 -0
- analyzer/subscription.py +108 -0
- burn_rate.py +75 -0
- cache_quarantine.py +50 -0
- codex_disk_cache.py +227 -0
- codex_events.py +136 -0
- codex_fork_replay.py +111 -0
- codex_loader.py +1426 -0
- codex_paths.py +20 -0
- critter_frames.py +26 -0
- discussion_bridge.py +1196 -0
- discussion_cli.py +844 -0
- discussion_session.py +622 -0
- discussion_usage.py +13 -0
- discussion_window.py +955 -0
- disk_cache_common.py +132 -0
- disk_cache_lifecycle.py +39 -0
- doctor.py +452 -0
- fsevents_watch.py +207 -0
- history_disk_cache.py +110 -0
- history_loader.py +416 -0
- i18n.py +88 -0
- jsonl_limits.py +17 -0
- jsonl_utils.py +40 -0
- login_item.py +154 -0
- main.py +387 -0
- menubar.py +1201 -0
- menubar_actions.py +204 -0
- menubar_agy.py +193 -0
- menubar_chrome.py +156 -0
- menubar_menu.py +169 -0
- menubar_notify.py +102 -0
- menubar_popover.py +233 -0
- menubar_prefs.py +118 -0
- menubar_refresh.py +285 -0
- menubar_state.py +1200 -0
- menubar_title.py +157 -0
- menubar_update.py +123 -0
- panel_window.py +78 -0
- panel_window_state.py +159 -0
- panels/__init__.py +186 -0
- panels/base.py +83 -0
- panels/dynamic_height.py +140 -0
- panels/payload.py +178 -0
- panels/web_panel.py +513 -0
- panels/window_drag.py +56 -0
- prefs.py +44 -0
- pricing.py +452 -0
- project_resolver.py +112 -0
- service_status.py +383 -0
- session_hooks.py +1154 -0
- setup_app.py +171 -0
- setup_hook.py +1011 -0
- statusline_settings.py +160 -0
- talent_market_bridge.py +243 -0
- time_utils.py +24 -0
- tui.py +288 -0
- tui_sprite.py +206 -0
- ui/__init__.py +5 -0
- ui/html_report.py +923 -0
- ui/report_scripts.py +251 -0
- ui/report_styles.py +370 -0
- ui/tables.py +888 -0
- update_checker.py +156 -0
- update_gate.py +66 -0
- update_release_notes.py +49 -0
- usage_cli-0.29.32.data/data/share/usage/i18n.json +2427 -0
- usage_cli-0.29.32.dist-info/METADATA +223 -0
- usage_cli-0.29.32.dist-info/RECORD +109 -0
- usage_cli-0.29.32.dist-info/WHEEL +5 -0
- usage_cli-0.29.32.dist-info/entry_points.txt +3 -0
- usage_cli-0.29.32.dist-info/licenses/LICENSE +663 -0
- usage_cli-0.29.32.dist-info/top_level.txt +80 -0
- usage_cli.py +827 -0
- usage_client.py +487 -0
- usage_diagnosis_snapshot.py +143 -0
- usage_dir_sweeper.py +100 -0
- usage_lang.py +79 -0
- usage_logging.py +75 -0
- usage_notifications.py +96 -0
- usage_rate.py +97 -0
- usage_session_resume.py +913 -0
- usage_statusline.py +810 -0
- usage_statusline_agy.py +397 -0
- usage_statusline_forwarder.py +88 -0
- usage_terse_mode.py +223 -0
- usage_terse_reminder.py +151 -0
- win_login_item.py +53 -0
- window_keeper.py +264 -0
- windows_watch.py +443 -0
- wintray.py +2014 -0
- wintray_menu.py +136 -0
codex_loader.py
ADDED
|
@@ -0,0 +1,1426 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 lollapalooza <https://github.com/aqua5230>
|
|
3
|
+
#
|
|
4
|
+
# Part of "usage". Free software licensed under the GNU Affero General Public
|
|
5
|
+
# License v3.0 only; see the LICENSE file for full terms and the warranty disclaimer.
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import sqlite3
|
|
14
|
+
import time
|
|
15
|
+
from collections import OrderedDict
|
|
16
|
+
from collections.abc import Iterable
|
|
17
|
+
from contextlib import closing
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from datetime import UTC, datetime, timedelta
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from codex_disk_cache import (
|
|
24
|
+
flush_caches,
|
|
25
|
+
seed_caches,
|
|
26
|
+
)
|
|
27
|
+
from codex_events import (
|
|
28
|
+
_as_dict,
|
|
29
|
+
_as_int,
|
|
30
|
+
_as_optional_float,
|
|
31
|
+
_as_str,
|
|
32
|
+
_event_value,
|
|
33
|
+
_load_json_line,
|
|
34
|
+
_session_model,
|
|
35
|
+
_timestamp_from_log_ts,
|
|
36
|
+
_token_usage_from_payload,
|
|
37
|
+
_TokenUsage,
|
|
38
|
+
)
|
|
39
|
+
from codex_events import (
|
|
40
|
+
_SessionFileInfo as _SessionFileInfo,
|
|
41
|
+
)
|
|
42
|
+
from codex_events import (
|
|
43
|
+
_ThreadMetadata as _ThreadMetadata,
|
|
44
|
+
)
|
|
45
|
+
from codex_fork_replay import (
|
|
46
|
+
_common_prefix_length,
|
|
47
|
+
_fork_replay_lookup_key,
|
|
48
|
+
_raw_token_usage_sequence,
|
|
49
|
+
_ReplayCacheKey,
|
|
50
|
+
_ReplayLookupKey,
|
|
51
|
+
_token_usage_events_after_embedded_parent,
|
|
52
|
+
)
|
|
53
|
+
from codex_paths import codex_home
|
|
54
|
+
from disk_cache_lifecycle import (
|
|
55
|
+
flush_caches_if_due,
|
|
56
|
+
needs_cache_seed,
|
|
57
|
+
)
|
|
58
|
+
from disk_cache_lifecycle import (
|
|
59
|
+
flush_caches_on_terminate as _flush_caches_on_terminate,
|
|
60
|
+
)
|
|
61
|
+
from history_loader import UsageEntry
|
|
62
|
+
from jsonl_limits import read_bounded_jsonl_line
|
|
63
|
+
from project_resolver import resolve_project_name
|
|
64
|
+
from time_utils import parse_optional_iso8601_utc
|
|
65
|
+
|
|
66
|
+
logger = logging.getLogger(__name__)
|
|
67
|
+
|
|
68
|
+
# Must comfortably exceed a real user's total *.jsonl session count. A cap at
|
|
69
|
+
# or below that count means every load_entries() call evicts and re-parses
|
|
70
|
+
# files that were just cached last refresh (LRU thrashing) — measured 512
|
|
71
|
+
# capped at 809 real sessions into a permanent 17+ second full-reparse every
|
|
72
|
+
# single call, even with per-file incremental caching working correctly in
|
|
73
|
+
# isolation. Also backs _file_info_cache and _fork_replay_cache below, which
|
|
74
|
+
# share the same real-world file count.
|
|
75
|
+
_JSONL_CACHE_MAXSIZE = 4096
|
|
76
|
+
_RECENT_JSONL_SCAN_LIMIT = 30
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(slots=True)
|
|
80
|
+
class _JsonlParseState:
|
|
81
|
+
session_timestamp: str = ""
|
|
82
|
+
project: str = "unknown"
|
|
83
|
+
session_model: str = "unknown"
|
|
84
|
+
previous_usage: _TokenUsage | None = None
|
|
85
|
+
token_count_index: int = 0
|
|
86
|
+
|
|
87
|
+
def copy(self) -> _JsonlParseState:
|
|
88
|
+
return _JsonlParseState(
|
|
89
|
+
session_timestamp=self.session_timestamp,
|
|
90
|
+
project=self.project,
|
|
91
|
+
session_model=self.session_model,
|
|
92
|
+
previous_usage=self.previous_usage,
|
|
93
|
+
token_count_index=self.token_count_index,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@dataclass(slots=True)
|
|
98
|
+
class _JsonlCacheEntry:
|
|
99
|
+
mtime: float
|
|
100
|
+
size: int
|
|
101
|
+
replay_cache_key: _ReplayCacheKey
|
|
102
|
+
entries: list[UsageEntry]
|
|
103
|
+
confirmed_offset: int = 0
|
|
104
|
+
confirmed_prefix_digest: bytes = b""
|
|
105
|
+
state: _JsonlParseState = field(default_factory=_JsonlParseState)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(slots=True)
|
|
109
|
+
class _SqliteLogCache:
|
|
110
|
+
watermark: tuple[int, int, int] | None = None
|
|
111
|
+
entries: list[UsageEntry] = field(default_factory=list)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
_jsonl_cache: OrderedDict[Path, _JsonlCacheEntry] = OrderedDict()
|
|
115
|
+
_fork_replay_cache: OrderedDict[
|
|
116
|
+
Path,
|
|
117
|
+
tuple[_ReplayLookupKey, int | None, _ReplayCacheKey],
|
|
118
|
+
] = OrderedDict()
|
|
119
|
+
_file_info_cache: OrderedDict[
|
|
120
|
+
Path,
|
|
121
|
+
tuple[float, int, _SessionFileInfo],
|
|
122
|
+
] = OrderedDict()
|
|
123
|
+
_sqlite_log_cache = _SqliteLogCache()
|
|
124
|
+
|
|
125
|
+
_SqliteFileFingerprint = tuple[
|
|
126
|
+
tuple[str, tuple[int, int] | None],
|
|
127
|
+
tuple[str, tuple[int, int] | None],
|
|
128
|
+
]
|
|
129
|
+
_thread_metadata_cache_key: _SqliteFileFingerprint | None = None
|
|
130
|
+
_thread_metadata_cache: dict[str, _ThreadMetadata] = {}
|
|
131
|
+
_sqlite_rate_limits_cache_key: _SqliteFileFingerprint | None = None
|
|
132
|
+
_sqlite_rate_limits_rows_cache: list[tuple[Any, Any]] = []
|
|
133
|
+
|
|
134
|
+
SESSIONS_DIR = codex_home() / "sessions"
|
|
135
|
+
ARCHIVED_SESSIONS_DIR = codex_home() / "archived_sessions"
|
|
136
|
+
STATE_DB = codex_home() / "state_5.sqlite"
|
|
137
|
+
LOGS_DB = codex_home() / "logs_2.sqlite"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _readonly_sqlite_uri(path: Path) -> str:
|
|
141
|
+
"""Return a read-only SQLite URI that also accepts Windows drive paths."""
|
|
142
|
+
return f"{path.resolve().as_uri()}?mode=ro"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _sqlite_file_fingerprint(path: Path) -> _SqliteFileFingerprint:
|
|
146
|
+
def stat_key(candidate: Path) -> tuple[int, int] | None:
|
|
147
|
+
try:
|
|
148
|
+
stat = candidate.stat()
|
|
149
|
+
except OSError:
|
|
150
|
+
return None
|
|
151
|
+
return (stat.st_mtime_ns, stat.st_size)
|
|
152
|
+
|
|
153
|
+
wal_path = Path(f"{path}-wal")
|
|
154
|
+
return ((str(path), stat_key(path)), (str(wal_path), stat_key(wal_path)))
|
|
155
|
+
|
|
156
|
+
# Disk cache for JSONL parsing results. Schema version must be bumped when the
|
|
157
|
+
# serialization format or parsing logic changes incompatibly.
|
|
158
|
+
_CODEX_JSONL_CACHE_SCHEMA = 4
|
|
159
|
+
JSONL_CACHE_PATH = Path(os.path.expanduser("~/.usage/codex_jsonl_cache.json"))
|
|
160
|
+
|
|
161
|
+
# Module-level flag to ensure seed loading happens exactly once.
|
|
162
|
+
_disk_cache_seeded = False
|
|
163
|
+
_DISK_CACHE_FLUSH_INTERVAL_S = 300.0
|
|
164
|
+
_disk_cache_dirty = False
|
|
165
|
+
_last_disk_cache_flush_at: float | None = None
|
|
166
|
+
_monotonic = time.monotonic
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass(slots=True)
|
|
170
|
+
class CodexRateLimits:
|
|
171
|
+
five_hour_pct: float | None
|
|
172
|
+
five_hour_resets_at: float | None
|
|
173
|
+
seven_day_pct: float | None
|
|
174
|
+
seven_day_resets_at: float | None
|
|
175
|
+
# window length (minutes) Codex reports for each slot; drives the row label
|
|
176
|
+
# (≈300→Session, ≈10080→Weekly, ≈43200→Monthly). None when the source has no
|
|
177
|
+
# window_minutes (header/error fallbacks) or the slot is absent (free plan).
|
|
178
|
+
five_hour_window_minutes: float | None = None
|
|
179
|
+
seven_day_window_minutes: float | None = None
|
|
180
|
+
model: str | None = "unknown"
|
|
181
|
+
updated_at: str = ""
|
|
182
|
+
has_credits: bool = False
|
|
183
|
+
credit_balance: str | None = None
|
|
184
|
+
credits_unlimited: bool = False
|
|
185
|
+
limit_id: str = ""
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _seed_caches_from_disk() -> None:
|
|
189
|
+
"""Seed in-memory caches from disk exactly once. Silently fails on any error."""
|
|
190
|
+
global _disk_cache_seeded
|
|
191
|
+
|
|
192
|
+
if not needs_cache_seed(_disk_cache_seeded):
|
|
193
|
+
return
|
|
194
|
+
_disk_cache_seeded = True
|
|
195
|
+
seed_caches(
|
|
196
|
+
JSONL_CACHE_PATH,
|
|
197
|
+
_CODEX_JSONL_CACHE_SCHEMA,
|
|
198
|
+
_JSONL_CACHE_MAXSIZE,
|
|
199
|
+
_jsonl_cache,
|
|
200
|
+
_file_info_cache,
|
|
201
|
+
_sqlite_log_cache,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _flush_caches_to_disk(*, force: bool = False) -> None:
|
|
206
|
+
"""Atomically write current in-memory caches to disk."""
|
|
207
|
+
global _disk_cache_dirty, _last_disk_cache_flush_at
|
|
208
|
+
|
|
209
|
+
_disk_cache_dirty, _last_disk_cache_flush_at = flush_caches_if_due(
|
|
210
|
+
_disk_cache_dirty,
|
|
211
|
+
_last_disk_cache_flush_at,
|
|
212
|
+
_monotonic,
|
|
213
|
+
_DISK_CACHE_FLUSH_INTERVAL_S,
|
|
214
|
+
lambda: flush_caches(
|
|
215
|
+
JSONL_CACHE_PATH,
|
|
216
|
+
_CODEX_JSONL_CACHE_SCHEMA,
|
|
217
|
+
_jsonl_cache,
|
|
218
|
+
_file_info_cache,
|
|
219
|
+
_sqlite_log_cache,
|
|
220
|
+
),
|
|
221
|
+
force=force,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def flush_caches_on_terminate() -> None:
|
|
226
|
+
"""Best-effort persistence of cache changes still waiting for the throttle."""
|
|
227
|
+
_flush_caches_on_terminate(lambda: _flush_caches_to_disk(force=True))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def load_entries(
|
|
231
|
+
hours_back: int = 0,
|
|
232
|
+
*,
|
|
233
|
+
jsonl_paths: Iterable[Path] | None = None,
|
|
234
|
+
) -> list[UsageEntry]:
|
|
235
|
+
cutoff = datetime.now(UTC) - timedelta(hours=hours_back) if hours_back > 0 else None
|
|
236
|
+
metadata = _load_thread_metadata()
|
|
237
|
+
models = {session_id: data.model for session_id, data in metadata.items()}
|
|
238
|
+
entries = _load_jsonl_entries(SESSIONS_DIR, models, cutoff, jsonl_paths=jsonl_paths)
|
|
239
|
+
|
|
240
|
+
latest_jsonl_ts_by_session = {
|
|
241
|
+
entry.session_id: entry.timestamp
|
|
242
|
+
for entry in entries
|
|
243
|
+
}
|
|
244
|
+
entries.extend(_load_sqlite_log_entries(metadata, cutoff, latest_jsonl_ts_by_session))
|
|
245
|
+
entries.sort(key=lambda entry: entry.timestamp)
|
|
246
|
+
return entries
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _session_roots(primary_dir: Path) -> list[Path]:
|
|
250
|
+
roots = [primary_dir]
|
|
251
|
+
if ARCHIVED_SESSIONS_DIR.is_dir():
|
|
252
|
+
roots.append(ARCHIVED_SESSIONS_DIR)
|
|
253
|
+
return roots
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _session_root_for_path(path: Path) -> Path | None:
|
|
257
|
+
for root in (SESSIONS_DIR, ARCHIVED_SESSIONS_DIR):
|
|
258
|
+
if path.is_relative_to(root):
|
|
259
|
+
return root
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _load_jsonl_entries(
|
|
264
|
+
sessions_dir: Path,
|
|
265
|
+
models: dict[str, str],
|
|
266
|
+
cutoff: datetime | None,
|
|
267
|
+
*,
|
|
268
|
+
jsonl_paths: Iterable[Path] | None = None,
|
|
269
|
+
) -> list[UsageEntry]:
|
|
270
|
+
global _disk_cache_dirty
|
|
271
|
+
|
|
272
|
+
# Seed from disk on first call
|
|
273
|
+
_seed_caches_from_disk()
|
|
274
|
+
|
|
275
|
+
if jsonl_paths is None:
|
|
276
|
+
roots = [root for root in _session_roots(sessions_dir) if root.is_dir()]
|
|
277
|
+
if not roots:
|
|
278
|
+
return []
|
|
279
|
+
jsonl_path_list = [path for root in roots for path in root.rglob("*.jsonl")]
|
|
280
|
+
else:
|
|
281
|
+
jsonl_path_list = list(jsonl_paths)
|
|
282
|
+
if not jsonl_path_list:
|
|
283
|
+
return []
|
|
284
|
+
|
|
285
|
+
# Snapshot each cached file's (mtime, size) to detect new or re-parsed files.
|
|
286
|
+
# A re-parse overwrites an existing key in place, so cache size alone would
|
|
287
|
+
# miss an updated (still-growing) session file — exactly the active files we
|
|
288
|
+
# most want to persist. dict equality ignores LRU move_to_end reordering.
|
|
289
|
+
jsonl_snapshot = {str(p): (e.mtime, e.size) for p, e in _jsonl_cache.items()}
|
|
290
|
+
file_info_snapshot = {str(p): (v[0], v[1]) for p, v in _file_info_cache.items()}
|
|
291
|
+
|
|
292
|
+
entries_by_session: dict[str, list[UsageEntry]] = {}
|
|
293
|
+
cutoff_ts = cutoff.timestamp() if cutoff else None
|
|
294
|
+
file_info = {path: _read_session_file_info(path) for path in jsonl_path_list}
|
|
295
|
+
paths_by_session: dict[str, list[Path]] = {}
|
|
296
|
+
for path, info in file_info.items():
|
|
297
|
+
if info.session_id:
|
|
298
|
+
paths_by_session.setdefault(info.session_id, []).append(path)
|
|
299
|
+
|
|
300
|
+
for jsonl_path in jsonl_path_list:
|
|
301
|
+
if cutoff_ts is not None:
|
|
302
|
+
try:
|
|
303
|
+
if jsonl_path.stat().st_mtime < cutoff_ts:
|
|
304
|
+
continue
|
|
305
|
+
except OSError as exc:
|
|
306
|
+
logger.warning("failed to stat session log %s: %s", jsonl_path, exc)
|
|
307
|
+
continue
|
|
308
|
+
info = file_info[jsonl_path]
|
|
309
|
+
replay_boundary, replay_cache_key = _fork_replay_boundary(
|
|
310
|
+
jsonl_path,
|
|
311
|
+
info,
|
|
312
|
+
paths_by_session.get(info.forked_from_id, []),
|
|
313
|
+
)
|
|
314
|
+
parsed = _parse_jsonl(
|
|
315
|
+
jsonl_path,
|
|
316
|
+
models,
|
|
317
|
+
cutoff,
|
|
318
|
+
file_info=info,
|
|
319
|
+
replay_boundary=replay_boundary,
|
|
320
|
+
replay_cache_key=replay_cache_key,
|
|
321
|
+
)
|
|
322
|
+
if not parsed:
|
|
323
|
+
continue
|
|
324
|
+
existing = entries_by_session.get(parsed[0].session_id)
|
|
325
|
+
if existing is None or _is_better_session_log(parsed, existing):
|
|
326
|
+
entries_by_session[parsed[0].session_id] = parsed
|
|
327
|
+
|
|
328
|
+
# Flush to disk if any file was newly parsed or re-parsed (content changed)
|
|
329
|
+
if (
|
|
330
|
+
{str(p): (e.mtime, e.size) for p, e in _jsonl_cache.items()} != jsonl_snapshot
|
|
331
|
+
or {str(p): (v[0], v[1]) for p, v in _file_info_cache.items()} != file_info_snapshot
|
|
332
|
+
):
|
|
333
|
+
_disk_cache_dirty = True
|
|
334
|
+
_flush_caches_to_disk()
|
|
335
|
+
elif _disk_cache_dirty:
|
|
336
|
+
_flush_caches_to_disk()
|
|
337
|
+
|
|
338
|
+
return [
|
|
339
|
+
entry
|
|
340
|
+
for session_entries in entries_by_session.values()
|
|
341
|
+
for entry in session_entries
|
|
342
|
+
]
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _is_better_session_log(candidate: list[UsageEntry], existing: list[UsageEntry]) -> bool:
|
|
346
|
+
candidate_latest = candidate[-1]
|
|
347
|
+
existing_latest = existing[-1]
|
|
348
|
+
if candidate_latest.timestamp != existing_latest.timestamp:
|
|
349
|
+
return candidate_latest.timestamp > existing_latest.timestamp
|
|
350
|
+
return _session_total_tokens(candidate) > _session_total_tokens(existing)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _session_total_tokens(entries: list[UsageEntry]) -> int:
|
|
354
|
+
return sum(entry.total_tokens for entry in entries)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _read_session_file_info_uncached(path: Path) -> _SessionFileInfo:
|
|
358
|
+
try:
|
|
359
|
+
with path.open("rb") as file:
|
|
360
|
+
while True:
|
|
361
|
+
raw_line, too_long = read_bounded_jsonl_line(file)
|
|
362
|
+
if too_long:
|
|
363
|
+
logger.warning("skipping oversized JSONL line in codex session %s", path)
|
|
364
|
+
continue
|
|
365
|
+
if not raw_line:
|
|
366
|
+
break
|
|
367
|
+
data = _load_json_line(raw_line.decode("utf-8"))
|
|
368
|
+
if data is None or data.get("type") != "session_meta":
|
|
369
|
+
continue
|
|
370
|
+
payload = _as_dict(data.get("payload"))
|
|
371
|
+
return _SessionFileInfo(
|
|
372
|
+
session_id=_as_str(payload.get("id")),
|
|
373
|
+
forked_from_id=_as_str(payload.get("forked_from_id")),
|
|
374
|
+
)
|
|
375
|
+
except (OSError, UnicodeDecodeError):
|
|
376
|
+
return _SessionFileInfo()
|
|
377
|
+
return _SessionFileInfo()
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _read_session_file_info(path: Path) -> _SessionFileInfo:
|
|
381
|
+
try:
|
|
382
|
+
st = path.stat()
|
|
383
|
+
except OSError:
|
|
384
|
+
return _SessionFileInfo()
|
|
385
|
+
|
|
386
|
+
cached = _file_info_cache.get(path)
|
|
387
|
+
if cached is not None and cached[0] == st.st_mtime and cached[1] == st.st_size:
|
|
388
|
+
_file_info_cache.move_to_end(path)
|
|
389
|
+
return cached[2]
|
|
390
|
+
|
|
391
|
+
info = _read_session_file_info_uncached(path)
|
|
392
|
+
|
|
393
|
+
if path not in _file_info_cache and len(_file_info_cache) >= _JSONL_CACHE_MAXSIZE:
|
|
394
|
+
_file_info_cache.popitem(last=False)
|
|
395
|
+
_file_info_cache[path] = (st.st_mtime, st.st_size, info)
|
|
396
|
+
return info
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def load_rate_limits(
|
|
400
|
+
*,
|
|
401
|
+
jsonl_candidates: Iterable[tuple[Path, float]] | None = None,
|
|
402
|
+
) -> CodexRateLimits | None:
|
|
403
|
+
sqlite_limits = _load_sqlite_rate_limits()
|
|
404
|
+
jsonl_limits = _load_jsonl_rate_limits(jsonl_candidates=jsonl_candidates)
|
|
405
|
+
if sqlite_limits is None:
|
|
406
|
+
return jsonl_limits
|
|
407
|
+
if jsonl_limits is None:
|
|
408
|
+
return sqlite_limits
|
|
409
|
+
merged = _merge_rate_limits(sqlite_limits, jsonl_limits)
|
|
410
|
+
if merged is not None:
|
|
411
|
+
return merged
|
|
412
|
+
if _rate_limits_timestamp(jsonl_limits) > _rate_limits_timestamp(sqlite_limits):
|
|
413
|
+
return jsonl_limits
|
|
414
|
+
return sqlite_limits
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _load_jsonl_rate_limits(
|
|
418
|
+
*,
|
|
419
|
+
jsonl_candidates: Iterable[tuple[Path, float]] | None = None,
|
|
420
|
+
) -> CodexRateLimits | None:
|
|
421
|
+
if jsonl_candidates is None and not any(
|
|
422
|
+
root.is_dir() for root in _session_roots(SESSIONS_DIR)
|
|
423
|
+
):
|
|
424
|
+
return None
|
|
425
|
+
models = _load_thread_models()
|
|
426
|
+
# scan 30 recent sessions because short/interrupted Codex sessions write null rate_limits
|
|
427
|
+
fallback: CodexRateLimits | None = None
|
|
428
|
+
for path in _recent_jsonl_files(jsonl_candidates=jsonl_candidates):
|
|
429
|
+
rate_limits = _extract_rate_limits(path, models)
|
|
430
|
+
if rate_limits is None:
|
|
431
|
+
continue
|
|
432
|
+
if rate_limits.limit_id == "codex":
|
|
433
|
+
return rate_limits
|
|
434
|
+
if fallback is None:
|
|
435
|
+
fallback = rate_limits
|
|
436
|
+
return fallback
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _rate_limits_timestamp(rate_limits: CodexRateLimits) -> datetime:
|
|
440
|
+
parsed = _parse_timestamp(rate_limits.updated_at)
|
|
441
|
+
return parsed if parsed is not None else datetime.min.replace(tzinfo=UTC)
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _merge_rate_limits(
|
|
445
|
+
sqlite_limits: CodexRateLimits,
|
|
446
|
+
jsonl_limits: CodexRateLimits,
|
|
447
|
+
) -> CodexRateLimits | None:
|
|
448
|
+
sqlite_ts = _rate_limits_timestamp(sqlite_limits)
|
|
449
|
+
jsonl_ts = _rate_limits_timestamp(jsonl_limits)
|
|
450
|
+
five_pct, five_reset, five_window = _pick_rate_limit_window(
|
|
451
|
+
sqlite_limits.five_hour_pct,
|
|
452
|
+
sqlite_limits.five_hour_resets_at,
|
|
453
|
+
sqlite_limits.five_hour_window_minutes,
|
|
454
|
+
sqlite_ts,
|
|
455
|
+
jsonl_limits.five_hour_pct,
|
|
456
|
+
jsonl_limits.five_hour_resets_at,
|
|
457
|
+
jsonl_limits.five_hour_window_minutes,
|
|
458
|
+
jsonl_ts,
|
|
459
|
+
)
|
|
460
|
+
seven_pct, seven_reset, seven_window = _pick_rate_limit_window(
|
|
461
|
+
sqlite_limits.seven_day_pct,
|
|
462
|
+
sqlite_limits.seven_day_resets_at,
|
|
463
|
+
sqlite_limits.seven_day_window_minutes,
|
|
464
|
+
sqlite_ts,
|
|
465
|
+
jsonl_limits.seven_day_pct,
|
|
466
|
+
jsonl_limits.seven_day_resets_at,
|
|
467
|
+
jsonl_limits.seven_day_window_minutes,
|
|
468
|
+
jsonl_ts,
|
|
469
|
+
)
|
|
470
|
+
if five_pct is None and seven_pct is None:
|
|
471
|
+
return None
|
|
472
|
+
newer = jsonl_limits if jsonl_ts > sqlite_ts else sqlite_limits
|
|
473
|
+
return CodexRateLimits(
|
|
474
|
+
five_hour_pct=five_pct,
|
|
475
|
+
five_hour_resets_at=five_reset,
|
|
476
|
+
seven_day_pct=seven_pct,
|
|
477
|
+
seven_day_resets_at=seven_reset,
|
|
478
|
+
five_hour_window_minutes=five_window,
|
|
479
|
+
seven_day_window_minutes=seven_window,
|
|
480
|
+
model=newer.model,
|
|
481
|
+
updated_at=newer.updated_at,
|
|
482
|
+
has_credits=newer.has_credits,
|
|
483
|
+
credit_balance=newer.credit_balance,
|
|
484
|
+
credits_unlimited=newer.credits_unlimited,
|
|
485
|
+
limit_id=jsonl_limits.limit_id or sqlite_limits.limit_id,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _pick_rate_limit_window(
|
|
490
|
+
sqlite_pct: float | None,
|
|
491
|
+
sqlite_reset: float | None,
|
|
492
|
+
sqlite_window: float | None,
|
|
493
|
+
sqlite_ts: datetime,
|
|
494
|
+
jsonl_pct: float | None,
|
|
495
|
+
jsonl_reset: float | None,
|
|
496
|
+
jsonl_window: float | None,
|
|
497
|
+
jsonl_ts: datetime,
|
|
498
|
+
) -> tuple[float | None, float | None, float | None]:
|
|
499
|
+
if sqlite_pct is None:
|
|
500
|
+
return jsonl_pct, jsonl_reset, jsonl_window
|
|
501
|
+
if jsonl_pct is None:
|
|
502
|
+
return sqlite_pct, sqlite_reset, sqlite_window
|
|
503
|
+
if _active_window_limit_reached(sqlite_pct, sqlite_reset, jsonl_reset):
|
|
504
|
+
return sqlite_pct, sqlite_reset, sqlite_window
|
|
505
|
+
if jsonl_ts > sqlite_ts:
|
|
506
|
+
return jsonl_pct, jsonl_reset, jsonl_window
|
|
507
|
+
return sqlite_pct, sqlite_reset, sqlite_window
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _active_window_limit_reached(
|
|
511
|
+
sqlite_pct: float,
|
|
512
|
+
sqlite_reset: float | None,
|
|
513
|
+
jsonl_reset: float | None,
|
|
514
|
+
) -> bool:
|
|
515
|
+
if sqlite_pct < 100:
|
|
516
|
+
return False
|
|
517
|
+
if sqlite_reset is None:
|
|
518
|
+
return True
|
|
519
|
+
if sqlite_reset < datetime.now(UTC).timestamp():
|
|
520
|
+
return False
|
|
521
|
+
# A newer reset window means Codex has already moved past the 100% event.
|
|
522
|
+
return jsonl_reset is None or jsonl_reset <= sqlite_reset + 60
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _load_sqlite_rate_limits() -> CodexRateLimits | None:
|
|
526
|
+
global _sqlite_rate_limits_cache_key, _sqlite_rate_limits_rows_cache
|
|
527
|
+
|
|
528
|
+
fingerprint = _sqlite_file_fingerprint(LOGS_DB)
|
|
529
|
+
if fingerprint == _sqlite_rate_limits_cache_key:
|
|
530
|
+
rows = _sqlite_rate_limits_rows_cache
|
|
531
|
+
elif fingerprint[0][1] is None:
|
|
532
|
+
_sqlite_rate_limits_cache_key = fingerprint
|
|
533
|
+
_sqlite_rate_limits_rows_cache = []
|
|
534
|
+
return None
|
|
535
|
+
else:
|
|
536
|
+
query = (
|
|
537
|
+
"SELECT ts, feedback_log_body FROM logs "
|
|
538
|
+
"WHERE target = 'codex_api::endpoint::responses_websocket' "
|
|
539
|
+
"AND (feedback_log_body LIKE '%websocket event: {\"type\":\"codex.rate_limits\"%' "
|
|
540
|
+
"OR feedback_log_body LIKE "
|
|
541
|
+
"'%websocket event: {\"type\":\"error\"%usage_limit_reached%') "
|
|
542
|
+
"ORDER BY ts DESC, ts_nanos DESC, id DESC LIMIT 50"
|
|
543
|
+
)
|
|
544
|
+
try:
|
|
545
|
+
with closing(sqlite3.connect(_readonly_sqlite_uri(LOGS_DB), uri=True)) as conn:
|
|
546
|
+
rows = conn.execute(query).fetchall()
|
|
547
|
+
except (OSError, sqlite3.Error):
|
|
548
|
+
if os.environ.get("USAGE_DEBUG") == "1":
|
|
549
|
+
logger.warning("codex sqlite rate limits load failed", exc_info=True)
|
|
550
|
+
return None
|
|
551
|
+
_sqlite_rate_limits_cache_key = fingerprint
|
|
552
|
+
_sqlite_rate_limits_rows_cache = rows
|
|
553
|
+
|
|
554
|
+
for ts, body in rows:
|
|
555
|
+
parsed = _parse_sqlite_rate_limits_row(ts, body)
|
|
556
|
+
if parsed is not None:
|
|
557
|
+
return parsed
|
|
558
|
+
return None
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _parse_sqlite_rate_limits_row(ts: Any, body: Any) -> CodexRateLimits | None:
|
|
562
|
+
if not isinstance(body, str):
|
|
563
|
+
return None
|
|
564
|
+
event = _websocket_event_payload(body)
|
|
565
|
+
if not event:
|
|
566
|
+
return None
|
|
567
|
+
if event.get("type") == "codex.rate_limits":
|
|
568
|
+
return _rate_limits_from_websocket_event(event, body, ts)
|
|
569
|
+
if event.get("type") == "error":
|
|
570
|
+
return _rate_limits_from_websocket_error(event, body, ts)
|
|
571
|
+
return None
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _websocket_event_payload(body: str) -> dict[str, Any]:
|
|
575
|
+
marker = "websocket event: "
|
|
576
|
+
index = body.find(marker)
|
|
577
|
+
if index < 0:
|
|
578
|
+
return {}
|
|
579
|
+
try:
|
|
580
|
+
data = json.loads(body[index + len(marker):])
|
|
581
|
+
except json.JSONDecodeError:
|
|
582
|
+
return {}
|
|
583
|
+
return data if isinstance(data, dict) else {}
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def _rate_limits_from_websocket_event(
|
|
587
|
+
event: dict[str, Any],
|
|
588
|
+
body: str,
|
|
589
|
+
ts: Any,
|
|
590
|
+
) -> CodexRateLimits | None:
|
|
591
|
+
rate_limits = _as_dict(event.get("rate_limits"))
|
|
592
|
+
primary = _as_dict(rate_limits.get("primary"))
|
|
593
|
+
secondary = _as_dict(rate_limits.get("secondary"))
|
|
594
|
+
credits = _as_dict(rate_limits.get("credits"))
|
|
595
|
+
return _build_rate_limits(
|
|
596
|
+
primary_pct=_as_optional_float(primary.get("used_percent")),
|
|
597
|
+
primary_reset=_as_optional_float(primary.get("reset_at")),
|
|
598
|
+
secondary_pct=_as_optional_float(secondary.get("used_percent")),
|
|
599
|
+
secondary_reset=_as_optional_float(secondary.get("reset_at")),
|
|
600
|
+
primary_window_minutes=_as_optional_float(primary.get("window_minutes")),
|
|
601
|
+
secondary_window_minutes=_as_optional_float(secondary.get("window_minutes")),
|
|
602
|
+
has_credits=credits.get("has_credits") is True,
|
|
603
|
+
credit_balance=_as_str(credits.get("balance")) or None,
|
|
604
|
+
credits_unlimited=credits.get("unlimited") is True,
|
|
605
|
+
model=_event_value(body, "model") or "unknown",
|
|
606
|
+
updated_at=_timestamp_from_log_ts(ts),
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _rate_limits_from_websocket_error(
|
|
611
|
+
event: dict[str, Any],
|
|
612
|
+
body: str,
|
|
613
|
+
ts: Any,
|
|
614
|
+
) -> CodexRateLimits | None:
|
|
615
|
+
headers = _as_dict(event.get("headers"))
|
|
616
|
+
primary_reset = _as_optional_float(headers.get("X-Codex-Primary-Reset-At"))
|
|
617
|
+
secondary_reset = _as_optional_float(headers.get("X-Codex-Secondary-Reset-At"))
|
|
618
|
+
now_ts = datetime.now(UTC).timestamp()
|
|
619
|
+
if primary_reset is None:
|
|
620
|
+
primary_reset_after = _as_optional_float(headers.get("X-Codex-Primary-Reset-After-Seconds"))
|
|
621
|
+
primary_reset = now_ts + primary_reset_after if primary_reset_after is not None else None
|
|
622
|
+
if secondary_reset is None:
|
|
623
|
+
secondary_reset_after = _as_optional_float(
|
|
624
|
+
headers.get("X-Codex-Secondary-Reset-After-Seconds")
|
|
625
|
+
)
|
|
626
|
+
secondary_reset = (
|
|
627
|
+
now_ts + secondary_reset_after if secondary_reset_after is not None else None
|
|
628
|
+
)
|
|
629
|
+
return _build_rate_limits(
|
|
630
|
+
primary_pct=_as_optional_float(headers.get("X-Codex-Primary-Used-Percent")),
|
|
631
|
+
primary_reset=primary_reset,
|
|
632
|
+
secondary_pct=_as_optional_float(headers.get("X-Codex-Secondary-Used-Percent")),
|
|
633
|
+
secondary_reset=secondary_reset,
|
|
634
|
+
model=_event_value(body, "model") or "unknown",
|
|
635
|
+
updated_at=_timestamp_from_log_ts(ts),
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _build_rate_limits(
|
|
640
|
+
*,
|
|
641
|
+
primary_pct: float | None,
|
|
642
|
+
primary_reset: float | None,
|
|
643
|
+
secondary_pct: float | None,
|
|
644
|
+
secondary_reset: float | None,
|
|
645
|
+
model: str,
|
|
646
|
+
updated_at: datetime | None,
|
|
647
|
+
primary_window_minutes: float | None = None,
|
|
648
|
+
secondary_window_minutes: float | None = None,
|
|
649
|
+
has_credits: bool = False,
|
|
650
|
+
credit_balance: str | None = None,
|
|
651
|
+
credits_unlimited: bool = False,
|
|
652
|
+
) -> CodexRateLimits | None:
|
|
653
|
+
now_ts = datetime.now(UTC).timestamp()
|
|
654
|
+
if primary_reset is not None and primary_reset < now_ts:
|
|
655
|
+
primary_pct = None
|
|
656
|
+
primary_reset = None
|
|
657
|
+
if secondary_reset is not None and secondary_reset < now_ts:
|
|
658
|
+
secondary_pct = None
|
|
659
|
+
secondary_reset = None
|
|
660
|
+
if primary_pct is None and secondary_pct is None:
|
|
661
|
+
return None
|
|
662
|
+
(
|
|
663
|
+
primary_pct,
|
|
664
|
+
primary_reset,
|
|
665
|
+
primary_window_minutes,
|
|
666
|
+
secondary_pct,
|
|
667
|
+
secondary_reset,
|
|
668
|
+
secondary_window_minutes,
|
|
669
|
+
) = _assign_rate_limit_slots(
|
|
670
|
+
primary_pct,
|
|
671
|
+
primary_reset,
|
|
672
|
+
primary_window_minutes,
|
|
673
|
+
secondary_pct,
|
|
674
|
+
secondary_reset,
|
|
675
|
+
secondary_window_minutes,
|
|
676
|
+
)
|
|
677
|
+
return CodexRateLimits(
|
|
678
|
+
five_hour_pct=primary_pct,
|
|
679
|
+
five_hour_resets_at=primary_reset,
|
|
680
|
+
seven_day_pct=secondary_pct,
|
|
681
|
+
seven_day_resets_at=secondary_reset,
|
|
682
|
+
five_hour_window_minutes=primary_window_minutes,
|
|
683
|
+
seven_day_window_minutes=secondary_window_minutes,
|
|
684
|
+
model=model,
|
|
685
|
+
updated_at=updated_at.isoformat() if updated_at is not None else "",
|
|
686
|
+
has_credits=has_credits,
|
|
687
|
+
credit_balance=credit_balance,
|
|
688
|
+
credits_unlimited=credits_unlimited,
|
|
689
|
+
)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def _assign_rate_limit_slots(
|
|
693
|
+
primary_pct: float | None,
|
|
694
|
+
primary_reset: float | None,
|
|
695
|
+
primary_window_minutes: float | None,
|
|
696
|
+
secondary_pct: float | None,
|
|
697
|
+
secondary_reset: float | None,
|
|
698
|
+
secondary_window_minutes: float | None,
|
|
699
|
+
) -> tuple[
|
|
700
|
+
float | None,
|
|
701
|
+
float | None,
|
|
702
|
+
float | None,
|
|
703
|
+
float | None,
|
|
704
|
+
float | None,
|
|
705
|
+
float | None,
|
|
706
|
+
]:
|
|
707
|
+
primary_is_session = (
|
|
708
|
+
primary_window_minutes is not None and primary_window_minutes <= 600.0
|
|
709
|
+
)
|
|
710
|
+
secondary_is_session = (
|
|
711
|
+
secondary_window_minutes is not None and secondary_window_minutes <= 600.0
|
|
712
|
+
)
|
|
713
|
+
classify_by_window = (
|
|
714
|
+
primary_window_minutes is not None
|
|
715
|
+
and secondary_window_minutes is not None
|
|
716
|
+
and primary_is_session != secondary_is_session
|
|
717
|
+
) or (
|
|
718
|
+
primary_window_minutes is not None
|
|
719
|
+
and secondary_pct is None
|
|
720
|
+
and secondary_window_minutes is None
|
|
721
|
+
) or (
|
|
722
|
+
secondary_window_minutes is not None
|
|
723
|
+
and primary_pct is None
|
|
724
|
+
and primary_window_minutes is None
|
|
725
|
+
)
|
|
726
|
+
if classify_by_window and not primary_is_session:
|
|
727
|
+
primary_pct, secondary_pct = secondary_pct, primary_pct
|
|
728
|
+
primary_reset, secondary_reset = secondary_reset, primary_reset
|
|
729
|
+
primary_window_minutes, secondary_window_minutes = (
|
|
730
|
+
secondary_window_minutes,
|
|
731
|
+
primary_window_minutes,
|
|
732
|
+
)
|
|
733
|
+
return (
|
|
734
|
+
primary_pct,
|
|
735
|
+
primary_reset,
|
|
736
|
+
primary_window_minutes,
|
|
737
|
+
secondary_pct,
|
|
738
|
+
secondary_reset,
|
|
739
|
+
secondary_window_minutes,
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _load_thread_models() -> dict[str, str]:
|
|
744
|
+
return {
|
|
745
|
+
thread_id: metadata.model
|
|
746
|
+
for thread_id, metadata in _load_thread_metadata().items()
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _load_thread_metadata() -> dict[str, _ThreadMetadata]:
|
|
751
|
+
global _thread_metadata_cache, _thread_metadata_cache_key
|
|
752
|
+
|
|
753
|
+
fingerprint = _sqlite_file_fingerprint(STATE_DB)
|
|
754
|
+
if fingerprint == _thread_metadata_cache_key:
|
|
755
|
+
return _thread_metadata_cache
|
|
756
|
+
if fingerprint[0][1] is None:
|
|
757
|
+
_thread_metadata_cache_key = fingerprint
|
|
758
|
+
_thread_metadata_cache = {}
|
|
759
|
+
return {}
|
|
760
|
+
try:
|
|
761
|
+
with closing(sqlite3.connect(_readonly_sqlite_uri(STATE_DB), uri=True)) as conn:
|
|
762
|
+
rows = conn.execute(
|
|
763
|
+
"SELECT id, model, cwd FROM threads",
|
|
764
|
+
).fetchall()
|
|
765
|
+
except (OSError, sqlite3.Error):
|
|
766
|
+
if os.environ.get("USAGE_DEBUG") == "1":
|
|
767
|
+
logger.warning("codex thread metadata load failed", exc_info=True)
|
|
768
|
+
return {}
|
|
769
|
+
result = {
|
|
770
|
+
thread_id: _ThreadMetadata(
|
|
771
|
+
model=model if isinstance(model, str) and model else "unknown",
|
|
772
|
+
cwd=cwd if isinstance(cwd, str) else "",
|
|
773
|
+
)
|
|
774
|
+
for thread_id, model, cwd in rows
|
|
775
|
+
if isinstance(thread_id, str) and thread_id
|
|
776
|
+
}
|
|
777
|
+
_thread_metadata_cache_key = fingerprint
|
|
778
|
+
_thread_metadata_cache = result
|
|
779
|
+
return result
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
def _load_sqlite_log_entries(
|
|
783
|
+
metadata: dict[str, _ThreadMetadata],
|
|
784
|
+
cutoff: datetime | None,
|
|
785
|
+
latest_jsonl_ts_by_session: dict[str, datetime],
|
|
786
|
+
) -> list[UsageEntry]:
|
|
787
|
+
global _disk_cache_dirty
|
|
788
|
+
|
|
789
|
+
if not LOGS_DB.exists():
|
|
790
|
+
return []
|
|
791
|
+
# Codex CLI <=0.14x wrote token-usage rows under this target; current versions
|
|
792
|
+
# don't (verified 2026-08-10: zero hits, no gap vs jsonl on an up-to-date CLI).
|
|
793
|
+
# Kept for users still holding old-format history in logs_2.sqlite.
|
|
794
|
+
query = (
|
|
795
|
+
"SELECT id, ts, ts_nanos, feedback_log_body FROM logs "
|
|
796
|
+
"WHERE target = 'codex_otel.trace_safe' "
|
|
797
|
+
"AND feedback_log_body LIKE '%event.kind=response.completed%' "
|
|
798
|
+
"AND feedback_log_body LIKE '%input_token_count=%'"
|
|
799
|
+
)
|
|
800
|
+
params: tuple[int, ...] = ()
|
|
801
|
+
if _sqlite_log_cache.watermark is not None:
|
|
802
|
+
query += " AND (ts, ts_nanos, id) > (?, ?, ?)"
|
|
803
|
+
params = _sqlite_log_cache.watermark
|
|
804
|
+
query += " ORDER BY ts ASC, ts_nanos ASC, id ASC"
|
|
805
|
+
try:
|
|
806
|
+
with closing(sqlite3.connect(_readonly_sqlite_uri(LOGS_DB), uri=True)) as conn:
|
|
807
|
+
conn.execute("BEGIN")
|
|
808
|
+
rows = conn.execute(query, params).fetchall()
|
|
809
|
+
newest_rows = conn.execute(
|
|
810
|
+
"SELECT ts, ts_nanos, id FROM logs "
|
|
811
|
+
"ORDER BY ts DESC, ts_nanos DESC, id DESC LIMIT 1"
|
|
812
|
+
).fetchall()
|
|
813
|
+
newest = newest_rows[0] if newest_rows else None
|
|
814
|
+
except (OSError, sqlite3.Error):
|
|
815
|
+
if os.environ.get("USAGE_DEBUG") == "1":
|
|
816
|
+
logger.warning("codex sqlite logs load failed", exc_info=True)
|
|
817
|
+
return []
|
|
818
|
+
|
|
819
|
+
candidates = list(_sqlite_log_cache.entries)
|
|
820
|
+
for row_id, ts, ts_nanos, body in rows:
|
|
821
|
+
entry = _parse_sqlite_log_row(row_id, ts, ts_nanos, body, metadata)
|
|
822
|
+
if entry is not None:
|
|
823
|
+
candidates.append(entry)
|
|
824
|
+
|
|
825
|
+
watermark = _sqlite_log_watermark(newest)
|
|
826
|
+
if _sqlite_log_cache.watermark is not None and (
|
|
827
|
+
watermark is None or watermark < _sqlite_log_cache.watermark
|
|
828
|
+
):
|
|
829
|
+
watermark = _sqlite_log_cache.watermark
|
|
830
|
+
if watermark != _sqlite_log_cache.watermark or len(candidates) != len(
|
|
831
|
+
_sqlite_log_cache.entries
|
|
832
|
+
):
|
|
833
|
+
_sqlite_log_cache.watermark = watermark
|
|
834
|
+
_sqlite_log_cache.entries = candidates
|
|
835
|
+
_disk_cache_dirty = True
|
|
836
|
+
_flush_caches_to_disk()
|
|
837
|
+
|
|
838
|
+
entries: list[UsageEntry] = []
|
|
839
|
+
for entry in candidates:
|
|
840
|
+
if cutoff is not None and entry.timestamp < cutoff:
|
|
841
|
+
continue
|
|
842
|
+
latest_jsonl_ts = latest_jsonl_ts_by_session.get(entry.session_id)
|
|
843
|
+
if latest_jsonl_ts is not None and entry.timestamp <= latest_jsonl_ts:
|
|
844
|
+
continue
|
|
845
|
+
entries.append(entry)
|
|
846
|
+
return entries
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _sqlite_log_watermark(row: Any) -> tuple[int, int, int] | None:
|
|
850
|
+
if not isinstance(row, (list, tuple)) or len(row) != 3:
|
|
851
|
+
return None
|
|
852
|
+
try:
|
|
853
|
+
return int(row[0]), int(row[1]), int(row[2])
|
|
854
|
+
except (TypeError, ValueError):
|
|
855
|
+
return None
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _parse_sqlite_log_row(
|
|
859
|
+
row_id: Any,
|
|
860
|
+
ts: Any,
|
|
861
|
+
ts_nanos: Any,
|
|
862
|
+
body: Any,
|
|
863
|
+
metadata: dict[str, _ThreadMetadata],
|
|
864
|
+
) -> UsageEntry | None:
|
|
865
|
+
if not isinstance(body, str):
|
|
866
|
+
return None
|
|
867
|
+
if 'event.name="codex.sse_event"' not in body or "event.kind=response.completed" not in body:
|
|
868
|
+
return None
|
|
869
|
+
session_id = _event_value(body, "conversation.id")
|
|
870
|
+
if not session_id:
|
|
871
|
+
return None
|
|
872
|
+
timestamp = _parse_timestamp(_event_value(body, "event.timestamp"))
|
|
873
|
+
if timestamp is None:
|
|
874
|
+
timestamp = _timestamp_from_log_ts(ts)
|
|
875
|
+
if timestamp is None:
|
|
876
|
+
return None
|
|
877
|
+
cached = _as_int(_event_value(body, "cached_token_count"))
|
|
878
|
+
input_tokens = max(0, _as_int(_event_value(body, "input_token_count")) - cached)
|
|
879
|
+
output_tokens = _as_int(_event_value(body, "output_token_count"))
|
|
880
|
+
if input_tokens + output_tokens + cached == 0:
|
|
881
|
+
return None
|
|
882
|
+
thread = metadata.get(session_id, _ThreadMetadata())
|
|
883
|
+
model = _event_value(body, "model") or thread.model
|
|
884
|
+
project = _project_from_cwd(thread.cwd) if thread.cwd else "unknown"
|
|
885
|
+
return UsageEntry(
|
|
886
|
+
timestamp=timestamp,
|
|
887
|
+
session_id=session_id,
|
|
888
|
+
message_id=f"{session_id}:sqlite:{row_id}:{ts_nanos}",
|
|
889
|
+
request_id="",
|
|
890
|
+
model=model,
|
|
891
|
+
input_tokens=input_tokens,
|
|
892
|
+
output_tokens=output_tokens,
|
|
893
|
+
cache_creation_tokens=0,
|
|
894
|
+
cache_read_tokens=cached,
|
|
895
|
+
cost_usd=None,
|
|
896
|
+
project=project,
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _recent_jsonl_files(
|
|
901
|
+
*,
|
|
902
|
+
jsonl_candidates: Iterable[tuple[Path, float]] | None = None,
|
|
903
|
+
) -> list[Path]:
|
|
904
|
+
if jsonl_candidates is not None:
|
|
905
|
+
visible_candidates = [
|
|
906
|
+
(mtime, path)
|
|
907
|
+
for path, mtime in jsonl_candidates
|
|
908
|
+
if _is_visible_jsonl(path)
|
|
909
|
+
]
|
|
910
|
+
visible_candidates.sort(key=lambda item: item[0], reverse=True)
|
|
911
|
+
return [path for _, path in visible_candidates[:_RECENT_JSONL_SCAN_LIMIT]]
|
|
912
|
+
try:
|
|
913
|
+
paths = [
|
|
914
|
+
path
|
|
915
|
+
for root in _session_roots(SESSIONS_DIR)
|
|
916
|
+
for path in root.rglob("*.jsonl")
|
|
917
|
+
if _is_visible_jsonl(path)
|
|
918
|
+
]
|
|
919
|
+
except OSError:
|
|
920
|
+
return []
|
|
921
|
+
return _sort_recent_jsonl_files(paths)
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def _is_visible_jsonl(path: Path) -> bool:
|
|
925
|
+
root = _session_root_for_path(path)
|
|
926
|
+
if root is None:
|
|
927
|
+
return False
|
|
928
|
+
relative = path.relative_to(root)
|
|
929
|
+
return all(not part.startswith(".") for part in relative.parts)
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def _sort_recent_jsonl_files(paths: list[Path]) -> list[Path]:
|
|
933
|
+
paths_with_mtime: list[tuple[float, Path]] = []
|
|
934
|
+
for path in paths:
|
|
935
|
+
try:
|
|
936
|
+
paths_with_mtime.append((path.stat().st_mtime, path))
|
|
937
|
+
except OSError as exc:
|
|
938
|
+
logger.warning("failed to stat codex session %s: %s", path, exc)
|
|
939
|
+
paths_with_mtime.sort(key=lambda item: item[0], reverse=True)
|
|
940
|
+
return [path for _, path in paths_with_mtime[:_RECENT_JSONL_SCAN_LIMIT]]
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def _extract_rate_limits(path: Path, models: dict[str, str]) -> CodexRateLimits | None:
|
|
944
|
+
session_id = ""
|
|
945
|
+
session_model = "unknown"
|
|
946
|
+
last_rate_limits: tuple[dict[str, Any], str] | None = None
|
|
947
|
+
last_general_rate_limits: tuple[dict[str, Any], str] | None = None
|
|
948
|
+
try:
|
|
949
|
+
with path.open("rb") as file:
|
|
950
|
+
while True:
|
|
951
|
+
raw_line, too_long = read_bounded_jsonl_line(file)
|
|
952
|
+
if too_long:
|
|
953
|
+
logger.warning("skipping oversized JSONL line in codex session %s", path)
|
|
954
|
+
continue
|
|
955
|
+
if not raw_line:
|
|
956
|
+
break
|
|
957
|
+
data = _load_json_line(raw_line.decode("utf-8"))
|
|
958
|
+
if data is None:
|
|
959
|
+
continue
|
|
960
|
+
if data.get("type") == "session_meta":
|
|
961
|
+
session_id = _as_str(_as_dict(data.get("payload")).get("id"))
|
|
962
|
+
session_model = _session_model(data.get("payload"), session_model)
|
|
963
|
+
continue
|
|
964
|
+
if data.get("type") == "turn_context":
|
|
965
|
+
session_model = _session_model(data.get("payload"), session_model)
|
|
966
|
+
continue
|
|
967
|
+
if data.get("type") != "event_msg":
|
|
968
|
+
continue
|
|
969
|
+
payload = _as_dict(data.get("payload"))
|
|
970
|
+
if payload.get("type") != "token_count":
|
|
971
|
+
continue
|
|
972
|
+
rate_limits = _as_dict(payload.get("rate_limits"))
|
|
973
|
+
if rate_limits:
|
|
974
|
+
last_rate_limits = (rate_limits, _as_str(data.get("timestamp")))
|
|
975
|
+
if rate_limits.get("limit_id") == "codex":
|
|
976
|
+
last_general_rate_limits = last_rate_limits
|
|
977
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
978
|
+
logger.warning("failed to read codex session %s: %s", path, exc)
|
|
979
|
+
return None
|
|
980
|
+
last_rate_limits = last_general_rate_limits or last_rate_limits
|
|
981
|
+
if last_rate_limits is None:
|
|
982
|
+
return None
|
|
983
|
+
rate_limits, updated_at = last_rate_limits
|
|
984
|
+
primary = _as_dict(rate_limits.get("primary"))
|
|
985
|
+
secondary = _as_dict(rate_limits.get("secondary"))
|
|
986
|
+
credits = _as_dict(rate_limits.get("credits"))
|
|
987
|
+
five_pct = _as_optional_float(primary.get("used_percent"))
|
|
988
|
+
five_reset = _as_optional_float(primary.get("resets_at"))
|
|
989
|
+
five_window = _as_optional_float(primary.get("window_minutes"))
|
|
990
|
+
seven_pct = _as_optional_float(secondary.get("used_percent"))
|
|
991
|
+
seven_reset = _as_optional_float(secondary.get("resets_at"))
|
|
992
|
+
seven_window = _as_optional_float(secondary.get("window_minutes"))
|
|
993
|
+
now_ts = datetime.now(UTC).timestamp()
|
|
994
|
+
if five_reset is not None and five_reset < now_ts:
|
|
995
|
+
five_pct = 0.0
|
|
996
|
+
five_reset = None
|
|
997
|
+
if seven_reset is not None and seven_reset < now_ts:
|
|
998
|
+
seven_pct = 0.0
|
|
999
|
+
seven_reset = None
|
|
1000
|
+
if five_pct is None and seven_pct is None:
|
|
1001
|
+
return None
|
|
1002
|
+
five_pct, five_reset, five_window, seven_pct, seven_reset, seven_window = (
|
|
1003
|
+
_assign_rate_limit_slots(
|
|
1004
|
+
five_pct,
|
|
1005
|
+
five_reset,
|
|
1006
|
+
five_window,
|
|
1007
|
+
seven_pct,
|
|
1008
|
+
seven_reset,
|
|
1009
|
+
seven_window,
|
|
1010
|
+
)
|
|
1011
|
+
)
|
|
1012
|
+
return CodexRateLimits(
|
|
1013
|
+
five_hour_pct=five_pct,
|
|
1014
|
+
five_hour_resets_at=five_reset,
|
|
1015
|
+
seven_day_pct=seven_pct,
|
|
1016
|
+
seven_day_resets_at=seven_reset,
|
|
1017
|
+
five_hour_window_minutes=five_window,
|
|
1018
|
+
seven_day_window_minutes=seven_window,
|
|
1019
|
+
model=models.get(session_id, session_model),
|
|
1020
|
+
updated_at=updated_at,
|
|
1021
|
+
has_credits=credits.get("has_credits") is True,
|
|
1022
|
+
credit_balance=_as_str(credits.get("balance")) or None,
|
|
1023
|
+
credits_unlimited=credits.get("unlimited") is True,
|
|
1024
|
+
limit_id=_as_str(rate_limits.get("limit_id")),
|
|
1025
|
+
)
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def _fork_replay_boundary(
|
|
1029
|
+
path: Path,
|
|
1030
|
+
info: _SessionFileInfo,
|
|
1031
|
+
parent_paths: list[Path],
|
|
1032
|
+
) -> tuple[int | None, _ReplayCacheKey]:
|
|
1033
|
+
if not info.forked_from_id:
|
|
1034
|
+
return 0, None
|
|
1035
|
+
|
|
1036
|
+
# Fork logs rewrite replay timestamps, but preserve the parent's cumulative token sequence.
|
|
1037
|
+
lookup_key = _fork_replay_lookup_key(path, parent_paths)
|
|
1038
|
+
if lookup_key is None:
|
|
1039
|
+
return None, None
|
|
1040
|
+
cached = _fork_replay_cache.get(path)
|
|
1041
|
+
if cached is not None and cached[0] == lookup_key:
|
|
1042
|
+
_fork_replay_cache.move_to_end(path)
|
|
1043
|
+
return cached[1], cached[2]
|
|
1044
|
+
|
|
1045
|
+
child_events = _token_usage_events_after_embedded_parent(path, info.forked_from_id)
|
|
1046
|
+
if child_events is None:
|
|
1047
|
+
result: tuple[int | None, _ReplayCacheKey] = (0, None)
|
|
1048
|
+
_cache_fork_replay_boundary(path, lookup_key, result)
|
|
1049
|
+
return result
|
|
1050
|
+
if not lookup_key[2]:
|
|
1051
|
+
result = (None, None)
|
|
1052
|
+
_cache_fork_replay_boundary(path, lookup_key, result)
|
|
1053
|
+
return result
|
|
1054
|
+
|
|
1055
|
+
child_usage = [usage for _, usage in child_events]
|
|
1056
|
+
best_match = 0
|
|
1057
|
+
best_key: _ReplayCacheKey = None
|
|
1058
|
+
for parent_path in parent_paths:
|
|
1059
|
+
match_count = _common_prefix_length(
|
|
1060
|
+
child_usage,
|
|
1061
|
+
_raw_token_usage_sequence(parent_path),
|
|
1062
|
+
)
|
|
1063
|
+
if match_count <= best_match:
|
|
1064
|
+
continue
|
|
1065
|
+
try:
|
|
1066
|
+
parent_stat = parent_path.stat()
|
|
1067
|
+
except OSError:
|
|
1068
|
+
continue
|
|
1069
|
+
best_match = match_count
|
|
1070
|
+
best_key = (
|
|
1071
|
+
str(parent_path),
|
|
1072
|
+
parent_stat.st_mtime,
|
|
1073
|
+
parent_stat.st_size,
|
|
1074
|
+
match_count,
|
|
1075
|
+
)
|
|
1076
|
+
|
|
1077
|
+
if child_events and best_match == 0:
|
|
1078
|
+
result = (None, None)
|
|
1079
|
+
_cache_fork_replay_boundary(path, lookup_key, result)
|
|
1080
|
+
return result
|
|
1081
|
+
boundary = child_events[best_match - 1][0] if best_match else 0
|
|
1082
|
+
result = (boundary, best_key)
|
|
1083
|
+
_cache_fork_replay_boundary(path, lookup_key, result)
|
|
1084
|
+
return result
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
def _cache_fork_replay_boundary(
|
|
1088
|
+
path: Path,
|
|
1089
|
+
lookup_key: _ReplayLookupKey,
|
|
1090
|
+
result: tuple[int | None, _ReplayCacheKey],
|
|
1091
|
+
) -> None:
|
|
1092
|
+
if path not in _fork_replay_cache and len(_fork_replay_cache) >= _JSONL_CACHE_MAXSIZE:
|
|
1093
|
+
_fork_replay_cache.popitem(last=False)
|
|
1094
|
+
_fork_replay_cache[path] = (lookup_key, result[0], result[1])
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def _cache_jsonl_entry(path: Path, entry: _JsonlCacheEntry) -> None:
|
|
1098
|
+
if path not in _jsonl_cache and len(_jsonl_cache) >= _JSONL_CACHE_MAXSIZE:
|
|
1099
|
+
_jsonl_cache.popitem(last=False)
|
|
1100
|
+
_jsonl_cache[path] = entry
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
def _confirmed_prefix_hasher(path: Path, cached: _JsonlCacheEntry) -> Any | None:
|
|
1104
|
+
if cached.confirmed_offset == 0:
|
|
1105
|
+
return hashlib.blake2b(digest_size=16)
|
|
1106
|
+
digest = hashlib.blake2b(digest_size=16)
|
|
1107
|
+
remaining = cached.confirmed_offset
|
|
1108
|
+
try:
|
|
1109
|
+
with path.open("rb") as file:
|
|
1110
|
+
while remaining > 0:
|
|
1111
|
+
chunk = file.read(min(remaining, 65536))
|
|
1112
|
+
if not chunk:
|
|
1113
|
+
return None
|
|
1114
|
+
digest.update(chunk)
|
|
1115
|
+
remaining -= len(chunk)
|
|
1116
|
+
except OSError:
|
|
1117
|
+
return None
|
|
1118
|
+
if digest.digest() != cached.confirmed_prefix_digest:
|
|
1119
|
+
return None
|
|
1120
|
+
return digest
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def _parse_linear_jsonl_bytes(
|
|
1124
|
+
file: Any,
|
|
1125
|
+
*,
|
|
1126
|
+
session_id: str,
|
|
1127
|
+
models: dict[str, str],
|
|
1128
|
+
entries: list[UsageEntry],
|
|
1129
|
+
state: _JsonlParseState,
|
|
1130
|
+
digest: Any,
|
|
1131
|
+
confirmed_offset: int,
|
|
1132
|
+
) -> int:
|
|
1133
|
+
while True:
|
|
1134
|
+
line_start = int(file.tell())
|
|
1135
|
+
line, too_long = read_bounded_jsonl_line(file)
|
|
1136
|
+
if too_long:
|
|
1137
|
+
logger.warning("skipping oversized JSONL line in codex session %s", session_id)
|
|
1138
|
+
confirmed_offset = int(file.tell())
|
|
1139
|
+
continue
|
|
1140
|
+
if not line:
|
|
1141
|
+
return confirmed_offset
|
|
1142
|
+
data = _load_json_line(line.decode("utf-8", errors="replace"))
|
|
1143
|
+
if not line.endswith(b"\n") and data is None:
|
|
1144
|
+
return line_start
|
|
1145
|
+
digest.update(line)
|
|
1146
|
+
confirmed_offset = int(file.tell())
|
|
1147
|
+
if data is None:
|
|
1148
|
+
continue
|
|
1149
|
+
if data.get("type") == "session_meta":
|
|
1150
|
+
payload = _as_dict(data.get("payload"))
|
|
1151
|
+
if not state.session_timestamp:
|
|
1152
|
+
state.session_timestamp = _as_str(payload.get("timestamp"))
|
|
1153
|
+
state.project = _project_from_cwd(_as_str(payload.get("cwd")))
|
|
1154
|
+
state.session_model = _session_model(payload, state.session_model)
|
|
1155
|
+
continue
|
|
1156
|
+
if data.get("type") == "turn_context":
|
|
1157
|
+
state.session_model = _session_model(data.get("payload"), state.session_model)
|
|
1158
|
+
continue
|
|
1159
|
+
if data.get("type") != "event_msg":
|
|
1160
|
+
continue
|
|
1161
|
+
payload = _as_dict(data.get("payload"))
|
|
1162
|
+
if payload.get("type") != "token_count":
|
|
1163
|
+
continue
|
|
1164
|
+
usage = _as_dict(_as_dict(payload.get("info")).get("total_token_usage"))
|
|
1165
|
+
timestamp = _parse_timestamp(_as_str(data.get("timestamp")))
|
|
1166
|
+
if not usage or not session_id or timestamp is None:
|
|
1167
|
+
continue
|
|
1168
|
+
current_usage = _token_usage_from_payload(usage)
|
|
1169
|
+
delta = current_usage.delta(state.previous_usage)
|
|
1170
|
+
state.previous_usage = current_usage
|
|
1171
|
+
if delta.total_tokens == 0:
|
|
1172
|
+
continue
|
|
1173
|
+
state.token_count_index += 1
|
|
1174
|
+
entries.append(
|
|
1175
|
+
UsageEntry(
|
|
1176
|
+
timestamp=timestamp,
|
|
1177
|
+
session_id=session_id,
|
|
1178
|
+
message_id=f"{session_id}:{state.token_count_index}",
|
|
1179
|
+
request_id="",
|
|
1180
|
+
model=models.get(session_id, state.session_model),
|
|
1181
|
+
input_tokens=delta.input_tokens,
|
|
1182
|
+
output_tokens=delta.output_tokens,
|
|
1183
|
+
cache_creation_tokens=0,
|
|
1184
|
+
cache_read_tokens=delta.cache_read_tokens,
|
|
1185
|
+
cost_usd=None,
|
|
1186
|
+
project=state.project,
|
|
1187
|
+
)
|
|
1188
|
+
)
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def _refresh_linear_jsonl_cache(
|
|
1192
|
+
path: Path,
|
|
1193
|
+
st: os.stat_result,
|
|
1194
|
+
session_id: str,
|
|
1195
|
+
models: dict[str, str],
|
|
1196
|
+
cached: _JsonlCacheEntry | None,
|
|
1197
|
+
) -> _JsonlCacheEntry | None:
|
|
1198
|
+
prefix_hasher = (
|
|
1199
|
+
_confirmed_prefix_hasher(path, cached)
|
|
1200
|
+
if cached is not None and st.st_size >= cached.confirmed_offset and st.st_size > cached.size
|
|
1201
|
+
else None
|
|
1202
|
+
)
|
|
1203
|
+
if prefix_hasher is not None:
|
|
1204
|
+
assert cached is not None
|
|
1205
|
+
incremental_entries = list(cached.entries)
|
|
1206
|
+
state = cached.state.copy()
|
|
1207
|
+
try:
|
|
1208
|
+
with path.open("rb") as file:
|
|
1209
|
+
file.seek(cached.confirmed_offset)
|
|
1210
|
+
confirmed_offset = _parse_linear_jsonl_bytes(
|
|
1211
|
+
file,
|
|
1212
|
+
session_id=session_id,
|
|
1213
|
+
models=models,
|
|
1214
|
+
entries=incremental_entries,
|
|
1215
|
+
state=state,
|
|
1216
|
+
digest=prefix_hasher,
|
|
1217
|
+
confirmed_offset=cached.confirmed_offset,
|
|
1218
|
+
)
|
|
1219
|
+
except OSError as exc:
|
|
1220
|
+
logger.warning("failed to parse codex session %s: %s", path, exc)
|
|
1221
|
+
return None
|
|
1222
|
+
return _JsonlCacheEntry(
|
|
1223
|
+
mtime=st.st_mtime,
|
|
1224
|
+
size=st.st_size,
|
|
1225
|
+
replay_cache_key=None,
|
|
1226
|
+
entries=incremental_entries,
|
|
1227
|
+
confirmed_offset=confirmed_offset,
|
|
1228
|
+
confirmed_prefix_digest=prefix_hasher.digest(),
|
|
1229
|
+
state=state,
|
|
1230
|
+
)
|
|
1231
|
+
|
|
1232
|
+
entries: list[UsageEntry] = []
|
|
1233
|
+
state = _JsonlParseState()
|
|
1234
|
+
digest = hashlib.blake2b(digest_size=16)
|
|
1235
|
+
try:
|
|
1236
|
+
with path.open("rb") as file:
|
|
1237
|
+
confirmed_offset = _parse_linear_jsonl_bytes(
|
|
1238
|
+
file,
|
|
1239
|
+
session_id=session_id,
|
|
1240
|
+
models=models,
|
|
1241
|
+
entries=entries,
|
|
1242
|
+
state=state,
|
|
1243
|
+
digest=digest,
|
|
1244
|
+
confirmed_offset=0,
|
|
1245
|
+
)
|
|
1246
|
+
except OSError as exc:
|
|
1247
|
+
logger.warning("failed to parse codex session %s: %s", path, exc)
|
|
1248
|
+
return None
|
|
1249
|
+
return _JsonlCacheEntry(
|
|
1250
|
+
mtime=st.st_mtime,
|
|
1251
|
+
size=st.st_size,
|
|
1252
|
+
replay_cache_key=None,
|
|
1253
|
+
entries=entries,
|
|
1254
|
+
confirmed_offset=confirmed_offset,
|
|
1255
|
+
confirmed_prefix_digest=digest.digest(),
|
|
1256
|
+
state=state,
|
|
1257
|
+
)
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
def _parse_jsonl(
|
|
1261
|
+
path: Path,
|
|
1262
|
+
models: dict[str, str],
|
|
1263
|
+
cutoff: datetime | None,
|
|
1264
|
+
*,
|
|
1265
|
+
file_info: _SessionFileInfo | None = None,
|
|
1266
|
+
replay_boundary: int | None = 0,
|
|
1267
|
+
replay_cache_key: _ReplayCacheKey = None,
|
|
1268
|
+
) -> list[UsageEntry]:
|
|
1269
|
+
try:
|
|
1270
|
+
st = path.stat()
|
|
1271
|
+
except OSError as exc:
|
|
1272
|
+
logger.warning("failed to parse codex session %s: %s", path, exc)
|
|
1273
|
+
return []
|
|
1274
|
+
|
|
1275
|
+
cache_entry = _jsonl_cache.get(path)
|
|
1276
|
+
if (
|
|
1277
|
+
cache_entry is not None
|
|
1278
|
+
and cache_entry.mtime == st.st_mtime
|
|
1279
|
+
and cache_entry.size == st.st_size
|
|
1280
|
+
and cache_entry.replay_cache_key == replay_cache_key
|
|
1281
|
+
):
|
|
1282
|
+
_jsonl_cache.move_to_end(path)
|
|
1283
|
+
cached_entries = cache_entry.entries
|
|
1284
|
+
for entry in cached_entries:
|
|
1285
|
+
if entry.session_id in models:
|
|
1286
|
+
entry.model = models[entry.session_id]
|
|
1287
|
+
if cutoff is None:
|
|
1288
|
+
return cached_entries
|
|
1289
|
+
return [entry for entry in cached_entries if entry.timestamp >= cutoff]
|
|
1290
|
+
|
|
1291
|
+
info = file_info or _read_session_file_info(path)
|
|
1292
|
+
if replay_boundary is None:
|
|
1293
|
+
return []
|
|
1294
|
+
|
|
1295
|
+
if not info.forked_from_id and replay_boundary == 0 and replay_cache_key is None:
|
|
1296
|
+
refreshed = _refresh_linear_jsonl_cache(path, st, info.session_id, models, cache_entry)
|
|
1297
|
+
if refreshed is None:
|
|
1298
|
+
_cache_jsonl_entry(
|
|
1299
|
+
path,
|
|
1300
|
+
_JsonlCacheEntry(
|
|
1301
|
+
mtime=st.st_mtime,
|
|
1302
|
+
size=st.st_size,
|
|
1303
|
+
replay_cache_key=None,
|
|
1304
|
+
entries=[],
|
|
1305
|
+
),
|
|
1306
|
+
)
|
|
1307
|
+
return []
|
|
1308
|
+
_cache_jsonl_entry(path, refreshed)
|
|
1309
|
+
refreshed_entries = refreshed.entries
|
|
1310
|
+
# Carried-forward entries from a prior incremental parse were built
|
|
1311
|
+
# before `models` (thread->model, resolved from sqlite) may have caught
|
|
1312
|
+
# up — reapply it here too, not just to entries parsed in this call, or
|
|
1313
|
+
# a stale/unknown model (and thus wrong per-model cost) sticks forever.
|
|
1314
|
+
for entry in refreshed_entries:
|
|
1315
|
+
if entry.session_id in models:
|
|
1316
|
+
entry.model = models[entry.session_id]
|
|
1317
|
+
if cutoff is not None:
|
|
1318
|
+
return [entry for entry in refreshed_entries if entry.timestamp >= cutoff]
|
|
1319
|
+
return refreshed_entries
|
|
1320
|
+
|
|
1321
|
+
session_id = info.session_id
|
|
1322
|
+
session_timestamp = ""
|
|
1323
|
+
project = "unknown"
|
|
1324
|
+
session_model = "unknown"
|
|
1325
|
+
entries: list[UsageEntry] = []
|
|
1326
|
+
previous_usage: _TokenUsage | None = None
|
|
1327
|
+
token_count_index = 0
|
|
1328
|
+
try:
|
|
1329
|
+
with path.open("rb") as file:
|
|
1330
|
+
line_number = 0
|
|
1331
|
+
while True:
|
|
1332
|
+
raw_line, too_long = read_bounded_jsonl_line(file)
|
|
1333
|
+
if not raw_line and not too_long:
|
|
1334
|
+
break
|
|
1335
|
+
line_number += 1
|
|
1336
|
+
if too_long:
|
|
1337
|
+
logger.warning("skipping oversized JSONL line in codex session %s", path)
|
|
1338
|
+
continue
|
|
1339
|
+
data = _load_json_line(raw_line.decode("utf-8"))
|
|
1340
|
+
if data is None:
|
|
1341
|
+
continue
|
|
1342
|
+
if data.get("type") == "session_meta":
|
|
1343
|
+
payload = _as_dict(data.get("payload"))
|
|
1344
|
+
if not session_timestamp:
|
|
1345
|
+
session_timestamp = _as_str(payload.get("timestamp"))
|
|
1346
|
+
project = _project_from_cwd(_as_str(payload.get("cwd")))
|
|
1347
|
+
session_model = _session_model(payload, session_model)
|
|
1348
|
+
continue
|
|
1349
|
+
if line_number <= replay_boundary:
|
|
1350
|
+
continue
|
|
1351
|
+
if data.get("type") == "turn_context":
|
|
1352
|
+
session_model = _session_model(data.get("payload"), session_model)
|
|
1353
|
+
continue
|
|
1354
|
+
if data.get("type") != "event_msg":
|
|
1355
|
+
continue
|
|
1356
|
+
payload = _as_dict(data.get("payload"))
|
|
1357
|
+
if payload.get("type") != "token_count":
|
|
1358
|
+
continue
|
|
1359
|
+
usage = _as_dict(_as_dict(payload.get("info")).get("total_token_usage"))
|
|
1360
|
+
timestamp = _parse_timestamp(_as_str(data.get("timestamp")))
|
|
1361
|
+
if not usage or not session_id or timestamp is None:
|
|
1362
|
+
continue
|
|
1363
|
+
current_usage = _token_usage_from_payload(usage)
|
|
1364
|
+
delta = current_usage.delta(previous_usage)
|
|
1365
|
+
previous_usage = current_usage
|
|
1366
|
+
if delta.total_tokens == 0:
|
|
1367
|
+
continue
|
|
1368
|
+
token_count_index += 1
|
|
1369
|
+
entries.append(
|
|
1370
|
+
UsageEntry(
|
|
1371
|
+
timestamp=timestamp,
|
|
1372
|
+
session_id=session_id,
|
|
1373
|
+
message_id=f"{session_id}:{token_count_index}",
|
|
1374
|
+
request_id="",
|
|
1375
|
+
model=models.get(session_id, session_model),
|
|
1376
|
+
input_tokens=delta.input_tokens,
|
|
1377
|
+
output_tokens=delta.output_tokens,
|
|
1378
|
+
cache_creation_tokens=0,
|
|
1379
|
+
cache_read_tokens=delta.cache_read_tokens,
|
|
1380
|
+
cost_usd=None,
|
|
1381
|
+
project=project,
|
|
1382
|
+
)
|
|
1383
|
+
)
|
|
1384
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
1385
|
+
logger.warning("failed to parse codex session %s: %s", path, exc)
|
|
1386
|
+
_cache_jsonl_entry(
|
|
1387
|
+
path,
|
|
1388
|
+
_JsonlCacheEntry(
|
|
1389
|
+
mtime=st.st_mtime,
|
|
1390
|
+
size=st.st_size,
|
|
1391
|
+
replay_cache_key=replay_cache_key,
|
|
1392
|
+
entries=[],
|
|
1393
|
+
),
|
|
1394
|
+
)
|
|
1395
|
+
return []
|
|
1396
|
+
if not entries and session_timestamp:
|
|
1397
|
+
_cache_jsonl_entry(
|
|
1398
|
+
path,
|
|
1399
|
+
_JsonlCacheEntry(
|
|
1400
|
+
mtime=st.st_mtime,
|
|
1401
|
+
size=st.st_size,
|
|
1402
|
+
replay_cache_key=replay_cache_key,
|
|
1403
|
+
entries=[],
|
|
1404
|
+
),
|
|
1405
|
+
)
|
|
1406
|
+
return []
|
|
1407
|
+
_cache_jsonl_entry(
|
|
1408
|
+
path,
|
|
1409
|
+
_JsonlCacheEntry(
|
|
1410
|
+
mtime=st.st_mtime,
|
|
1411
|
+
size=st.st_size,
|
|
1412
|
+
replay_cache_key=replay_cache_key,
|
|
1413
|
+
entries=entries,
|
|
1414
|
+
),
|
|
1415
|
+
)
|
|
1416
|
+
if cutoff is not None:
|
|
1417
|
+
return [entry for entry in entries if entry.timestamp >= cutoff]
|
|
1418
|
+
return entries
|
|
1419
|
+
|
|
1420
|
+
|
|
1421
|
+
def _parse_timestamp(value: Any) -> datetime | None:
|
|
1422
|
+
return parse_optional_iso8601_utc(value)
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def _project_from_cwd(cwd: str) -> str:
|
|
1426
|
+
return resolve_project_name(cwd)
|