cli-consumption 0.0.3__tar.gz → 0.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/PKG-INFO +31 -3
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/README.md +30 -2
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/docs/architecture.md +2 -2
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/docs/provider-support.md +33 -5
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/docs/roadmap.md +2 -1
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/pyproject.toml +1 -1
- cli_consumption-0.0.5/src/cli_consumption/adapters/__init__.py +6 -0
- cli_consumption-0.0.5/src/cli_consumption/adapters/opencode.py +463 -0
- cli_consumption-0.0.5/src/cli_consumption/adapters/pi.py +454 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/cli.py +19 -6
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/test_cli.py +82 -1
- cli_consumption-0.0.5/tests/test_opencode_adapter.py +261 -0
- cli_consumption-0.0.5/tests/test_pi_adapter.py +287 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/uv.lock +1 -1
- cli_consumption-0.0.3/src/cli_consumption/adapters/__init__.py +0 -4
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/add-cli-adapter/SKILL.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/add-cli-adapter/agents/openai.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/audit-usage-privacy/SKILL.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/audit-usage-privacy/agents/openai.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/evolve-storage-schema/SKILL.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/evolve-storage-schema/agents/openai.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/yeet-github/SKILL.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/yeet-github/agents/openai.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/yolo/SKILL.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.agents/skills/yolo/agents/openai.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.github/workflows/ci.yml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.github/workflows/release.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.gitignore +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.pre-commit-config.yaml +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/.python-version +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/AGENTS.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/CONTRIBUTING.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/LICENSE +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/NOTICE +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/docs/privacy.md +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/__init__.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/__main__.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/adapters/base.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/adapters/claude.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/adapters/codex.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/api.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/dashboard.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/exporting.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/models.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/py.typed +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/storage.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/src/cli_consumption/sync.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/conftest.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/test_api.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/test_claude_adapter.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/test_codex_adapter.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/test_storage_and_exports.py +0 -0
- {cli_consumption-0.0.3 → cli_consumption-0.0.5}/tests/test_sync.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cli-consumption
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.5
|
|
4
4
|
Summary: Analyze and consolidate AI coding CLI consumption across machines.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Guillaume-Lombardo/cli-consumption
|
|
6
6
|
Project-URL: Documentation, https://github.com/Guillaume-Lombardo/cli-consumption#readme
|
|
@@ -33,8 +33,8 @@ models, tokens, tools, conversations, and turns. It can analyze one workstation,
|
|
|
33
33
|
consolidate copied data from several machines, or send metadata-only snapshots to a
|
|
34
34
|
central collector.
|
|
35
35
|
|
|
36
|
-
Codex and the core Claude Code local transcript format are supported.
|
|
37
|
-
Code
|
|
36
|
+
Codex, Pi, OpenCode, and the core Claude Code local transcript format are supported.
|
|
37
|
+
Kilo Code is planned behind the same provider-neutral adapter contract.
|
|
38
38
|
|
|
39
39
|
The collector deliberately excludes prompts, responses, tool arguments, and
|
|
40
40
|
credentials. See [Privacy](docs/privacy.md) before sharing a database or export.
|
|
@@ -102,6 +102,18 @@ Select Claude Code to read `~/.claude/projects/` instead:
|
|
|
102
102
|
uv run cli-consumption collect --provider claude --database usage.sqlite
|
|
103
103
|
```
|
|
104
104
|
|
|
105
|
+
Select OpenCode to read `~/.local/share/opencode/opencode.db` instead:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
uv run cli-consumption collect --provider opencode --database usage.sqlite
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Select Pi to read `~/.pi/agent/sessions/` instead:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
uv run cli-consumption collect --provider pi --database usage.sqlite
|
|
115
|
+
```
|
|
116
|
+
|
|
105
117
|
Use `all` to detect and collect every supported provider present on the machine:
|
|
106
118
|
|
|
107
119
|
```bash
|
|
@@ -142,6 +154,22 @@ uv run cli-consumption collect --provider claude \
|
|
|
142
154
|
--source laptop=/data/claude/laptop
|
|
143
155
|
```
|
|
144
156
|
|
|
157
|
+
Copied OpenCode sources point to the data directory containing `opencode.db`:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
uv run cli-consumption collect --provider opencode \
|
|
161
|
+
--source desktop=/data/opencode/desktop \
|
|
162
|
+
--source laptop=/data/opencode/laptop
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Copied Pi sources point to the agent directory containing `sessions/`:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
uv run cli-consumption collect --provider pi \
|
|
169
|
+
--source desktop=/data/pi/desktop \
|
|
170
|
+
--source laptop=/data/pi/laptop
|
|
171
|
+
```
|
|
172
|
+
|
|
145
173
|
## SQLite and PostgreSQL
|
|
146
174
|
|
|
147
175
|
A file path selects SQLite. A SQLAlchemy URL selects PostgreSQL:
|
|
@@ -5,8 +5,8 @@ models, tokens, tools, conversations, and turns. It can analyze one workstation,
|
|
|
5
5
|
consolidate copied data from several machines, or send metadata-only snapshots to a
|
|
6
6
|
central collector.
|
|
7
7
|
|
|
8
|
-
Codex and the core Claude Code local transcript format are supported.
|
|
9
|
-
Code
|
|
8
|
+
Codex, Pi, OpenCode, and the core Claude Code local transcript format are supported.
|
|
9
|
+
Kilo Code is planned behind the same provider-neutral adapter contract.
|
|
10
10
|
|
|
11
11
|
The collector deliberately excludes prompts, responses, tool arguments, and
|
|
12
12
|
credentials. See [Privacy](docs/privacy.md) before sharing a database or export.
|
|
@@ -74,6 +74,18 @@ Select Claude Code to read `~/.claude/projects/` instead:
|
|
|
74
74
|
uv run cli-consumption collect --provider claude --database usage.sqlite
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
+
Select OpenCode to read `~/.local/share/opencode/opencode.db` instead:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
uv run cli-consumption collect --provider opencode --database usage.sqlite
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Select Pi to read `~/.pi/agent/sessions/` instead:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv run cli-consumption collect --provider pi --database usage.sqlite
|
|
87
|
+
```
|
|
88
|
+
|
|
77
89
|
Use `all` to detect and collect every supported provider present on the machine:
|
|
78
90
|
|
|
79
91
|
```bash
|
|
@@ -114,6 +126,22 @@ uv run cli-consumption collect --provider claude \
|
|
|
114
126
|
--source laptop=/data/claude/laptop
|
|
115
127
|
```
|
|
116
128
|
|
|
129
|
+
Copied OpenCode sources point to the data directory containing `opencode.db`:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
uv run cli-consumption collect --provider opencode \
|
|
133
|
+
--source desktop=/data/opencode/desktop \
|
|
134
|
+
--source laptop=/data/opencode/laptop
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Copied Pi sources point to the agent directory containing `sessions/`:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
uv run cli-consumption collect --provider pi \
|
|
141
|
+
--source desktop=/data/pi/desktop \
|
|
142
|
+
--source laptop=/data/pi/laptop
|
|
143
|
+
```
|
|
144
|
+
|
|
117
145
|
## SQLite and PostgreSQL
|
|
118
146
|
|
|
119
147
|
A file path selects SQLite. A SQLAlchemy URL selects PostgreSQL:
|
|
@@ -16,8 +16,8 @@ provider files -> adapter -> metadata-only snapshot -> SQL storage -> dashboard/
|
|
|
16
16
|
|
|
17
17
|
- `adapters`: parse a CLI's local data into conversations, turns, model calls, tool
|
|
18
18
|
calls, context-pressure samples, bounded turn settings, compactions, and content-free
|
|
19
|
-
work-item intervals. Codex exposes the complete analytics contract; Claude Code
|
|
20
|
-
|
|
19
|
+
work-item intervals. Codex exposes the complete analytics contract; Claude Code and
|
|
20
|
+
OpenCode expose the core dimensions available in their local stores.
|
|
21
21
|
- `models`: define the transport boundary shared by offline and API ingestion.
|
|
22
22
|
- `storage`: owns the normalized schema, idempotent replacement rules, SQLite, and
|
|
23
23
|
PostgreSQL engine creation.
|
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
| --- | --- | --- |
|
|
5
5
|
| Codex | Supported | Local rollout JSONL and optional metadata-only subagent state |
|
|
6
6
|
| Claude Code | Supported (core) | Local project transcript JSONL |
|
|
7
|
-
| OpenCode |
|
|
7
|
+
| OpenCode | Supported (core) | Local SQLite v2 session store |
|
|
8
|
+
| Pi | Supported (core) | Local session JSONL v1-v3 |
|
|
8
9
|
| Kilo Code | Planned | To be verified before implementation |
|
|
9
|
-
| Pi | Planned | To be verified before implementation |
|
|
10
10
|
|
|
11
11
|
“Supported” means the adapter has synthetic fixtures, extracts conversations, turns,
|
|
12
12
|
models, token usage, and tool names when available, and passes privacy regression tests.
|
|
@@ -36,6 +36,34 @@ is not billing data. This first increment does not collect subagent transcripts,
|
|
|
36
36
|
context-window sizes, effort/service-tier settings, TTFT, provider-reported duration,
|
|
37
37
|
or technical work-item intervals.
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
39
|
+
OpenCode reads `opencode.db` from its XDG data directory (normally
|
|
40
|
+
`~/.local/share/opencode/`). It extracts v2 session messages, model references, token
|
|
41
|
+
usage, tool names, and compaction timestamps while discarding message text, reasoning,
|
|
42
|
+
tool inputs/results, shell commands/output, paths, titles, errors, costs, and arbitrary
|
|
43
|
+
metadata. Model labels combine OpenCode's provider and model identifiers.
|
|
44
|
+
|
|
45
|
+
OpenCode reports uncached input, cache reads, cache writes, visible output, and
|
|
46
|
+
reasoning separately. Normalized input and output totals include their respective
|
|
47
|
+
components. The adapter does not currently read pre-v2 JSON storage, legacy
|
|
48
|
+
`message`/`part` tables, child-session relationships, context-window sizes, or
|
|
49
|
+
provider-reported cost. The SQLite schema is internal and may change without notice;
|
|
50
|
+
local token events are not billing data.
|
|
51
|
+
|
|
52
|
+
Pi reads session JSONL files under `~/.pi/agent/sessions/` (or copied agent
|
|
53
|
+
directories). It extracts user turns, assistant and compaction model calls, provider
|
|
54
|
+
and model identifiers, token usage, tool names, thinking-level changes, and compaction
|
|
55
|
+
timestamps while discarding prompts, responses, thinking, tool arguments/results,
|
|
56
|
+
commands/output, summaries, errors, costs, paths, extension data, and arbitrary
|
|
57
|
+
metadata. All persisted branches are counted because Pi retains branch history in the
|
|
58
|
+
same session file.
|
|
59
|
+
|
|
60
|
+
Pi reports uncached input, cache reads, cache writes, output, and optional reasoning
|
|
61
|
+
tokens. Normalized input includes all three input components; reasoning is retained as
|
|
62
|
+
a subset of output rather than added twice. This adapter does not collect custom
|
|
63
|
+
session directories automatically, context-window sizes, costs, branch relationships,
|
|
64
|
+
or provider-reported durations. Pi's JSONL schema can change without notice, and its
|
|
65
|
+
local token events are not billing data.
|
|
66
|
+
|
|
67
|
+
Provider formats can change without notice. Unknown fields are ignored; malformed
|
|
68
|
+
provider records are counted and skipped. Compatibility fixes should add a fixture for
|
|
69
|
+
both the old and new format whenever possible.
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
- Complete Codex local collector
|
|
6
6
|
- Core Claude Code local transcript collector
|
|
7
|
+
- Core Pi local session collector
|
|
7
8
|
- Offline multi-machine deduplication
|
|
8
9
|
- SQLite and PostgreSQL storage
|
|
9
10
|
- Metadata-only central collection API
|
|
@@ -14,7 +15,7 @@
|
|
|
14
15
|
|
|
15
16
|
1. Extend Claude Code support only where local metadata has reliable semantics.
|
|
16
17
|
2. Add OpenCode with the same normalized contract.
|
|
17
|
-
3. Add Kilo Code
|
|
18
|
+
3. Add Kilo Code after format and licensing review.
|
|
18
19
|
4. Add cross-provider comparison views once at least two adapters expose reliable model
|
|
19
20
|
and token dimensions.
|
|
20
21
|
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from cli_consumption.adapters.claude import ClaudeAdapter
|
|
2
|
+
from cli_consumption.adapters.codex import CodexAdapter
|
|
3
|
+
from cli_consumption.adapters.opencode import OpenCodeAdapter
|
|
4
|
+
from cli_consumption.adapters.pi import PiAdapter
|
|
5
|
+
|
|
6
|
+
__all__ = ["ClaudeAdapter", "CodexAdapter", "OpenCodeAdapter", "PiAdapter"]
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import math
|
|
6
|
+
import re
|
|
7
|
+
import sqlite3
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import UTC, datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from cli_consumption.models import Snapshot, empty_tokens
|
|
14
|
+
|
|
15
|
+
MAX_BIGINT = 9_223_372_036_854_775_807
|
|
16
|
+
SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/+-]*")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(slots=True)
|
|
20
|
+
class _Message:
|
|
21
|
+
external_id: str
|
|
22
|
+
kind: str
|
|
23
|
+
sequence: int
|
|
24
|
+
created_at: datetime | None
|
|
25
|
+
updated_at: datetime | None
|
|
26
|
+
data: dict[str, Any]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(slots=True)
|
|
30
|
+
class _Conversation:
|
|
31
|
+
machine: str
|
|
32
|
+
external_id: str
|
|
33
|
+
directory: str | None
|
|
34
|
+
created_at: datetime | None
|
|
35
|
+
updated_at: datetime | None
|
|
36
|
+
messages: list[_Message]
|
|
37
|
+
digest: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class OpenCodeAdapter:
|
|
41
|
+
"""Read metadata from OpenCode's SQLite v2 store without retaining content."""
|
|
42
|
+
|
|
43
|
+
name = "opencode"
|
|
44
|
+
|
|
45
|
+
def collect(
|
|
46
|
+
self,
|
|
47
|
+
sources: list[tuple[str, Path]],
|
|
48
|
+
project_mappings: list[tuple[str, str]] | None = None,
|
|
49
|
+
) -> Snapshot:
|
|
50
|
+
selected: dict[str, _Conversation] = {}
|
|
51
|
+
duplicates = malformed = 0
|
|
52
|
+
for machine, home in sources:
|
|
53
|
+
database = home / "opencode.db"
|
|
54
|
+
if not database.is_file():
|
|
55
|
+
raise ValueError(f"Missing OpenCode database: {database}")
|
|
56
|
+
conversations, invalid = _read_database(database, machine)
|
|
57
|
+
malformed += invalid
|
|
58
|
+
for candidate in conversations:
|
|
59
|
+
previous = selected.get(candidate.external_id)
|
|
60
|
+
if previous is None:
|
|
61
|
+
selected[candidate.external_id] = candidate
|
|
62
|
+
continue
|
|
63
|
+
duplicates += 1
|
|
64
|
+
if _rank(candidate) > _rank(previous):
|
|
65
|
+
selected[candidate.external_id] = candidate
|
|
66
|
+
|
|
67
|
+
snapshot = Snapshot(
|
|
68
|
+
provider=self.name,
|
|
69
|
+
duplicate_conversations=duplicates,
|
|
70
|
+
malformed_records=malformed,
|
|
71
|
+
)
|
|
72
|
+
for conversation in sorted(
|
|
73
|
+
selected.values(), key=lambda item: item.external_id
|
|
74
|
+
):
|
|
75
|
+
self._normalize(snapshot, conversation, project_mappings or [])
|
|
76
|
+
return snapshot
|
|
77
|
+
|
|
78
|
+
def _normalize(
|
|
79
|
+
self,
|
|
80
|
+
snapshot: Snapshot,
|
|
81
|
+
source: _Conversation,
|
|
82
|
+
mappings: list[tuple[str, str]],
|
|
83
|
+
) -> None:
|
|
84
|
+
conversation_id = f"opencode:{source.external_id}"
|
|
85
|
+
turns: dict[str, dict[str, Any]] = {}
|
|
86
|
+
turn_models: dict[str, set[str]] = {}
|
|
87
|
+
active: str | None = None
|
|
88
|
+
effective_model: str | None = None
|
|
89
|
+
calls: list[tuple[str | None, datetime | None, str, dict[str, int]]] = []
|
|
90
|
+
tools: list[tuple[str | None, datetime | None, str]] = []
|
|
91
|
+
compactions: list[tuple[str | None, datetime | None]] = []
|
|
92
|
+
timestamps = [source.created_at]
|
|
93
|
+
|
|
94
|
+
for message in source.messages:
|
|
95
|
+
timestamp = message.created_at or _timestamp(
|
|
96
|
+
_mapping(message.data.get("time")).get("created")
|
|
97
|
+
)
|
|
98
|
+
completed_at = _timestamp(
|
|
99
|
+
_mapping(message.data.get("time")).get("completed")
|
|
100
|
+
)
|
|
101
|
+
timestamps.extend((timestamp, completed_at))
|
|
102
|
+
|
|
103
|
+
if message.kind == "user":
|
|
104
|
+
if active is not None:
|
|
105
|
+
_finish_turn(turns[active], timestamp)
|
|
106
|
+
active = message.external_id
|
|
107
|
+
turns[active] = {
|
|
108
|
+
"id": f"{conversation_id}:{active}",
|
|
109
|
+
"conversation_id": conversation_id,
|
|
110
|
+
"external_id": active,
|
|
111
|
+
"started_at": _iso(timestamp),
|
|
112
|
+
"ended_at": None,
|
|
113
|
+
"status": "in-progress",
|
|
114
|
+
"duration_ms": None,
|
|
115
|
+
"time_to_first_token_ms": None,
|
|
116
|
+
"model_calls": 0,
|
|
117
|
+
"tool_calls": 0,
|
|
118
|
+
**empty_tokens(),
|
|
119
|
+
}
|
|
120
|
+
turn_models[active] = set()
|
|
121
|
+
continue
|
|
122
|
+
|
|
123
|
+
if message.kind == "model-switched":
|
|
124
|
+
effective_model = _model(message.data.get("model"))
|
|
125
|
+
continue
|
|
126
|
+
|
|
127
|
+
if message.kind == "compaction":
|
|
128
|
+
compactions.append((active, timestamp))
|
|
129
|
+
continue
|
|
130
|
+
|
|
131
|
+
if message.kind != "assistant":
|
|
132
|
+
continue
|
|
133
|
+
|
|
134
|
+
model = _model(message.data.get("model")) or effective_model or "unknown"
|
|
135
|
+
effective_model = model if model != "unknown" else effective_model
|
|
136
|
+
tokens = _usage(message.data.get("tokens"))
|
|
137
|
+
calls.append((active, timestamp, model, tokens))
|
|
138
|
+
turn = turns.get(active or "")
|
|
139
|
+
if turn:
|
|
140
|
+
turn["model_calls"] += 1
|
|
141
|
+
turn_models[active or ""].add(model)
|
|
142
|
+
_add_tokens(turn, tokens)
|
|
143
|
+
turn["ended_at"] = _iso(completed_at or timestamp) or turn["ended_at"]
|
|
144
|
+
if isinstance(message.data.get("error"), dict):
|
|
145
|
+
turn["status"] = "aborted"
|
|
146
|
+
elif (
|
|
147
|
+
completed_at is not None or _label(message.data.get("finish"), 255)
|
|
148
|
+
) and turn["status"] != "aborted":
|
|
149
|
+
turn["status"] = "completed"
|
|
150
|
+
|
|
151
|
+
content = message.data.get("content")
|
|
152
|
+
if not isinstance(content, list):
|
|
153
|
+
continue
|
|
154
|
+
for part in content:
|
|
155
|
+
if not isinstance(part, dict) or part.get("type") != "tool":
|
|
156
|
+
continue
|
|
157
|
+
name = _label(part.get("name"), 512)
|
|
158
|
+
if not name:
|
|
159
|
+
continue
|
|
160
|
+
part_time = _mapping(part.get("time"))
|
|
161
|
+
tools.append(
|
|
162
|
+
(active, _timestamp(part_time.get("created")) or timestamp, name)
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
ended_at = max(
|
|
166
|
+
(value for value in timestamps if value is not None), default=None
|
|
167
|
+
)
|
|
168
|
+
if active is not None:
|
|
169
|
+
_finish_turn(turns[active], ended_at or source.updated_at)
|
|
170
|
+
|
|
171
|
+
totals = empty_tokens()
|
|
172
|
+
models: set[str] = set()
|
|
173
|
+
for sequence, (turn_key, timestamp, model, tokens) in enumerate(calls, 1):
|
|
174
|
+
models.add(model)
|
|
175
|
+
_add_tokens(totals, tokens)
|
|
176
|
+
turn = turns.get(turn_key or "")
|
|
177
|
+
snapshot.model_calls.append(
|
|
178
|
+
{
|
|
179
|
+
"id": f"{conversation_id}:model:{sequence}",
|
|
180
|
+
"conversation_id": conversation_id,
|
|
181
|
+
"turn_id": turn["id"] if turn else None,
|
|
182
|
+
"sequence": sequence,
|
|
183
|
+
"timestamp": _iso(timestamp),
|
|
184
|
+
"model": model,
|
|
185
|
+
**tokens,
|
|
186
|
+
}
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
for sequence, (turn_key, timestamp, name) in enumerate(tools, 1):
|
|
190
|
+
turn = turns.get(turn_key or "")
|
|
191
|
+
if turn:
|
|
192
|
+
turn["tool_calls"] += 1
|
|
193
|
+
snapshot.tool_calls.append(
|
|
194
|
+
{
|
|
195
|
+
"id": f"{conversation_id}:tool:{sequence}",
|
|
196
|
+
"conversation_id": conversation_id,
|
|
197
|
+
"turn_id": turn["id"] if turn else None,
|
|
198
|
+
"sequence": sequence,
|
|
199
|
+
"timestamp": _iso(timestamp),
|
|
200
|
+
"tool_name": name,
|
|
201
|
+
"outer_tool_name": name,
|
|
202
|
+
}
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
for sequence, (turn_key, timestamp) in enumerate(compactions, 1):
|
|
206
|
+
turn = turns.get(turn_key or "")
|
|
207
|
+
snapshot.compaction_events.append(
|
|
208
|
+
{
|
|
209
|
+
"id": f"{conversation_id}:compaction:{sequence}",
|
|
210
|
+
"conversation_id": conversation_id,
|
|
211
|
+
"turn_id": turn["id"] if turn else None,
|
|
212
|
+
"sequence": sequence,
|
|
213
|
+
"timestamp": _iso(timestamp),
|
|
214
|
+
}
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
for key, turn in turns.items():
|
|
218
|
+
snapshot.turns.append(turn)
|
|
219
|
+
observed = turn_models[key]
|
|
220
|
+
snapshot.turn_settings.append(
|
|
221
|
+
{
|
|
222
|
+
"id": f"{conversation_id}:settings:{key}",
|
|
223
|
+
"conversation_id": conversation_id,
|
|
224
|
+
"turn_id": turn["id"],
|
|
225
|
+
"model": next(iter(observed)) if len(observed) == 1 else None,
|
|
226
|
+
"effort": None,
|
|
227
|
+
"collaboration_mode": None,
|
|
228
|
+
"service_tier": None,
|
|
229
|
+
"context_window_tokens": None,
|
|
230
|
+
}
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
started_at = source.created_at or min(
|
|
234
|
+
(value for value in timestamps if value is not None), default=None
|
|
235
|
+
)
|
|
236
|
+
ended_at = ended_at or source.updated_at
|
|
237
|
+
project, project_source = _project(source.directory, mappings)
|
|
238
|
+
snapshot.conversations.append(
|
|
239
|
+
{
|
|
240
|
+
"id": conversation_id,
|
|
241
|
+
"provider": self.name,
|
|
242
|
+
"external_id": source.external_id,
|
|
243
|
+
"source_machine": source.machine,
|
|
244
|
+
"project": project,
|
|
245
|
+
"project_source": project_source,
|
|
246
|
+
"started_at": _iso(started_at),
|
|
247
|
+
"ended_at": _iso(ended_at),
|
|
248
|
+
"duration_seconds": (
|
|
249
|
+
max(0.0, (ended_at - started_at).total_seconds())
|
|
250
|
+
if started_at and ended_at
|
|
251
|
+
else None
|
|
252
|
+
),
|
|
253
|
+
"source": "local-sqlite-v2",
|
|
254
|
+
"models": sorted(models),
|
|
255
|
+
"iterations": len(turns),
|
|
256
|
+
"model_calls": len(calls),
|
|
257
|
+
"tool_calls": len(tools),
|
|
258
|
+
"compactions": len(compactions),
|
|
259
|
+
"event_count": len(source.messages),
|
|
260
|
+
"content_hash": source.digest,
|
|
261
|
+
**totals,
|
|
262
|
+
}
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _read_database(path: Path, machine: str) -> tuple[list[_Conversation], int]:
|
|
267
|
+
try:
|
|
268
|
+
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
|
|
269
|
+
connection.row_factory = sqlite3.Row
|
|
270
|
+
connection.execute("PRAGMA trusted_schema=OFF")
|
|
271
|
+
connection.execute("PRAGMA query_only=ON")
|
|
272
|
+
session_columns = _columns(connection, "session")
|
|
273
|
+
message_columns = _columns(connection, "session_message")
|
|
274
|
+
required_session = {"id", "time_created", "time_updated"}
|
|
275
|
+
required_message = {
|
|
276
|
+
"id",
|
|
277
|
+
"session_id",
|
|
278
|
+
"type",
|
|
279
|
+
"seq",
|
|
280
|
+
"time_created",
|
|
281
|
+
"time_updated",
|
|
282
|
+
"data",
|
|
283
|
+
}
|
|
284
|
+
if (
|
|
285
|
+
not required_session <= session_columns
|
|
286
|
+
or not required_message <= message_columns
|
|
287
|
+
):
|
|
288
|
+
raise ValueError(f"Unsupported OpenCode database schema: {path}")
|
|
289
|
+
|
|
290
|
+
directory = "directory" if "directory" in session_columns else "NULL"
|
|
291
|
+
rows = connection.execute(
|
|
292
|
+
f"SELECT id, {directory} AS directory, time_created, time_updated "
|
|
293
|
+
"FROM session ORDER BY id"
|
|
294
|
+
).fetchall()
|
|
295
|
+
conversations: list[_Conversation] = []
|
|
296
|
+
malformed = 0
|
|
297
|
+
for row in rows:
|
|
298
|
+
external_id = _label(row["id"], 500)
|
|
299
|
+
if not external_id:
|
|
300
|
+
malformed += 1
|
|
301
|
+
continue
|
|
302
|
+
message_rows = connection.execute(
|
|
303
|
+
"SELECT id, type, seq, time_created, time_updated, data "
|
|
304
|
+
"FROM session_message WHERE session_id = ? ORDER BY seq, id",
|
|
305
|
+
(row["id"],),
|
|
306
|
+
).fetchall()
|
|
307
|
+
digest = hashlib.sha256()
|
|
308
|
+
messages: list[_Message] = []
|
|
309
|
+
for message_row in message_rows:
|
|
310
|
+
digest.update(str(tuple(message_row)).encode())
|
|
311
|
+
message_id = _label(message_row["id"], 512)
|
|
312
|
+
kind = _label(message_row["type"], 64)
|
|
313
|
+
try:
|
|
314
|
+
data = json.loads(message_row["data"])
|
|
315
|
+
except (json.JSONDecodeError, TypeError, UnicodeDecodeError):
|
|
316
|
+
malformed += 1
|
|
317
|
+
continue
|
|
318
|
+
if not message_id or not kind or not isinstance(data, dict):
|
|
319
|
+
malformed += 1
|
|
320
|
+
continue
|
|
321
|
+
sequence = _counter(message_row["seq"])
|
|
322
|
+
messages.append(
|
|
323
|
+
_Message(
|
|
324
|
+
message_id,
|
|
325
|
+
kind,
|
|
326
|
+
sequence,
|
|
327
|
+
_timestamp(message_row["time_created"]),
|
|
328
|
+
_timestamp(message_row["time_updated"]),
|
|
329
|
+
data,
|
|
330
|
+
)
|
|
331
|
+
)
|
|
332
|
+
conversations.append(
|
|
333
|
+
_Conversation(
|
|
334
|
+
machine=machine,
|
|
335
|
+
external_id=external_id,
|
|
336
|
+
directory=row["directory"]
|
|
337
|
+
if isinstance(row["directory"], str)
|
|
338
|
+
else None,
|
|
339
|
+
created_at=_timestamp(row["time_created"]),
|
|
340
|
+
updated_at=_timestamp(row["time_updated"]),
|
|
341
|
+
messages=messages,
|
|
342
|
+
digest=digest.hexdigest(),
|
|
343
|
+
)
|
|
344
|
+
)
|
|
345
|
+
return conversations, malformed
|
|
346
|
+
except sqlite3.DatabaseError:
|
|
347
|
+
raise ValueError(f"Could not read OpenCode database: {path}") from None
|
|
348
|
+
finally:
|
|
349
|
+
if "connection" in locals():
|
|
350
|
+
connection.close()
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _columns(connection: sqlite3.Connection, table: str) -> set[str]:
|
|
354
|
+
return {
|
|
355
|
+
str(row["name"]) for row in connection.execute(f'PRAGMA table_info("{table}")')
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _rank(value: _Conversation) -> tuple[int, datetime, str]:
|
|
360
|
+
return (
|
|
361
|
+
len(value.messages),
|
|
362
|
+
value.updated_at or datetime.min.replace(tzinfo=UTC),
|
|
363
|
+
value.digest,
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _usage(value: object) -> dict[str, int]:
|
|
368
|
+
usage = _mapping(value)
|
|
369
|
+
cache = _mapping(usage.get("cache"))
|
|
370
|
+
uncached = _counter(usage.get("input"))
|
|
371
|
+
cached = _counter(cache.get("read"))
|
|
372
|
+
cache_write = _counter(cache.get("write"))
|
|
373
|
+
visible = _counter(usage.get("output"))
|
|
374
|
+
reasoning = _counter(usage.get("reasoning"))
|
|
375
|
+
input_tokens = _sum(uncached, cached, cache_write)
|
|
376
|
+
output_tokens = _sum(visible, reasoning)
|
|
377
|
+
attributed = _sum(input_tokens, output_tokens)
|
|
378
|
+
reported = _counter(usage.get("total"))
|
|
379
|
+
total = max(attributed, reported)
|
|
380
|
+
return {
|
|
381
|
+
"input_tokens": input_tokens,
|
|
382
|
+
"cached_input_tokens": cached,
|
|
383
|
+
"cache_write_input_tokens": cache_write,
|
|
384
|
+
"output_tokens": output_tokens,
|
|
385
|
+
"reasoning_output_tokens": reasoning,
|
|
386
|
+
"total_tokens": total,
|
|
387
|
+
"uncached_input_tokens": uncached,
|
|
388
|
+
"visible_output_tokens": visible,
|
|
389
|
+
"unattributed_tokens": max(0, total - attributed),
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _model(value: object) -> str | None:
|
|
394
|
+
model = _mapping(value)
|
|
395
|
+
provider = _label(model.get("providerID"), 127)
|
|
396
|
+
identifier = _label(model.get("id") or model.get("modelID"), 127)
|
|
397
|
+
if provider and identifier:
|
|
398
|
+
return f"{provider}/{identifier}"
|
|
399
|
+
return identifier
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _project(directory: str | None, mappings: list[tuple[str, str]]) -> tuple[str, str]:
|
|
403
|
+
if directory:
|
|
404
|
+
normalized = directory.replace("\\", "/").rstrip("/")
|
|
405
|
+
for name, prefix in sorted(
|
|
406
|
+
mappings, key=lambda item: len(item[1]), reverse=True
|
|
407
|
+
):
|
|
408
|
+
prefix = prefix.replace("\\", "/").rstrip("/")
|
|
409
|
+
if normalized == prefix or normalized.startswith(prefix + "/"):
|
|
410
|
+
return name, "mapping"
|
|
411
|
+
return "outside-project", "none"
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _finish_turn(turn: dict[str, Any], fallback: datetime | None) -> None:
|
|
415
|
+
turn["ended_at"] = turn["ended_at"] or _iso(fallback)
|
|
416
|
+
start, end = _timestamp(turn["started_at"]), _timestamp(turn["ended_at"])
|
|
417
|
+
if start and end:
|
|
418
|
+
turn["duration_ms"] = max(0, int((end - start).total_seconds() * 1000))
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _mapping(value: object) -> dict[str, Any]:
|
|
422
|
+
return value if isinstance(value, dict) else {}
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _timestamp(value: object) -> datetime | None:
|
|
426
|
+
try:
|
|
427
|
+
if isinstance(value, bool):
|
|
428
|
+
return None
|
|
429
|
+
if isinstance(value, int | float) and math.isfinite(value):
|
|
430
|
+
return datetime.fromtimestamp(value / 1000, UTC)
|
|
431
|
+
if isinstance(value, str) and value:
|
|
432
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC)
|
|
433
|
+
except (OSError, OverflowError, ValueError):
|
|
434
|
+
pass
|
|
435
|
+
return None
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _iso(value: object) -> str | None:
|
|
439
|
+
return value.isoformat() if isinstance(value, datetime) else None
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _label(value: object, maximum: int) -> str | None:
|
|
443
|
+
if not isinstance(value, str):
|
|
444
|
+
return None
|
|
445
|
+
value = value.strip()
|
|
446
|
+
return value if 0 < len(value) <= maximum and SAFE_LABEL.fullmatch(value) else None
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _counter(value: object) -> int:
|
|
450
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
451
|
+
return 0
|
|
452
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
453
|
+
return 0
|
|
454
|
+
return min(MAX_BIGINT, max(0, int(value)))
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _sum(*values: int) -> int:
|
|
458
|
+
return min(MAX_BIGINT, sum(values))
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _add_tokens(target: dict[str, Any], tokens: dict[str, int]) -> None:
|
|
462
|
+
for field, value in tokens.items():
|
|
463
|
+
target[field] = _sum(int(target[field]), value)
|