context-forge-cli 0.1.0__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/PKG-INFO +7 -6
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/README.md +6 -5
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/altimate_code.py +59 -23
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/base.py +15 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/claude_code.py +77 -45
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/claude_desktop.py +13 -4
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/codex.py +95 -9
- context_forge_cli-0.2.5/contextforge/adapters/gemini.py +201 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/registry.py +2 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/cli.py +73 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/token_analyzer.py +23 -1
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/models/session.py +2 -0
- context_forge_cli-0.2.5/contextforge/tui/widgets/tokens_panel.py +290 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/pyproject.toml +1 -1
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/uv.lock +2 -2
- context_forge_cli-0.1.0/contextforge/tui/widgets/tokens_panel.py +0 -134
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/.contextforge.toml +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/.github/workflows/publish.yml +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/.gitignore +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/.python-version +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/AGENTS.md +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/PLAN.md +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/adapters/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/analytics.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/compactor.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/db.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/injector.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/scanner.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/core/summarizer.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/models/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/models/config.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/app.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/styles.tcss +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/widgets/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/widgets/session_detail.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/widgets/session_table.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/widgets/stats_panel.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/widgets/status_bar.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/tui/widgets/transfer_panel.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/utils/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/utils/display.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/contextforge/utils/tokens.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/adapters/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/adapters/test_claude_code.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/core/__init__.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/core/test_compactor.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/core/test_db.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/core/test_injector.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/core/test_summarizer.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/core/test_token_analyzer.py +0 -0
- {context_forge_cli-0.1.0 → context_forge_cli-0.2.5}/tests/fixtures/claude_session_sample.jsonl +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: context-forge-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Session manager and context bridge for agentic CLI tools
|
|
5
5
|
Project-URL: Homepage, https://github.com/emmver/contextforge
|
|
6
6
|
Project-URL: Repository, https://github.com/emmver/contextforge
|
|
@@ -30,14 +30,14 @@ Description-Content-Type: text/markdown
|
|
|
30
30
|
|
|
31
31
|
**Session manager and context bridge for agentic CLI tools.**
|
|
32
32
|
|
|
33
|
-
ContextForge (`cf`) tracks sessions from Claude Code, Codex, altimate-code, and other
|
|
33
|
+
ContextForge (`cf`) tracks sessions from Claude Code, Claude Desktop, Codex, Gemini, altimate-code, and other
|
|
34
34
|
agentic CLI tools. It generates plain-English summaries of each session and can compact
|
|
35
35
|
and transfer context from one or more sessions into a new session — within the same tool
|
|
36
36
|
or across different tools.
|
|
37
37
|
|
|
38
38
|
## What it does
|
|
39
39
|
|
|
40
|
-
- **Discovers sessions** from Claude Code (`~/.claude/projects/`), Codex (`~/.codex/state_5.sqlite`), and altimate-code (`~/.local/share/altimate-code/`)
|
|
40
|
+
- **Discovers sessions** from Claude Code (`~/.claude/projects/`), Claude Desktop, Codex (`~/.codex/state_5.sqlite`), Gemini (`~/.gemini/tmp/<project_hash>/chats/`), and altimate-code (`~/.local/share/altimate-code/`)
|
|
41
41
|
- **Generates summaries** of what each session accomplished (with optional Claude API integration)
|
|
42
42
|
- **Compacts context** intelligently, reducing multi-turn conversations to a token-efficient ContextBundle
|
|
43
43
|
- **Transfers context** across tools and sessions (Claude Code → Codex, Codex → altimate-code, etc.)
|
|
@@ -47,7 +47,7 @@ or across different tools.
|
|
|
47
47
|
|
|
48
48
|
- **Python 3.13+** (ContextForge requires Python 3.13 or later)
|
|
49
49
|
- **uv** or **pipx** (for installation as a tool)
|
|
50
|
-
- At least one of: Claude Code, Codex, or altimate-code installed and with session history
|
|
50
|
+
- At least one of: Claude Code, Claude Desktop, Codex, Gemini, or altimate-code installed and with session history
|
|
51
51
|
|
|
52
52
|
## Install
|
|
53
53
|
|
|
@@ -167,6 +167,7 @@ The method is chosen automatically based on token count and whether a target ses
|
|
|
167
167
|
| Claude Code (CLI) | `~/.claude/projects/` JSONL | `--system-prompt` / `--resume` |
|
|
168
168
|
| Claude Desktop | `~/Library/Application Support/Claude/local-agent-mode-sessions/` | `--system-prompt` |
|
|
169
169
|
| Codex | `~/.codex/state_5.sqlite` | `resume` / `fork` |
|
|
170
|
+
| Gemini | `~/.gemini/tmp/<project_hash>/chats/` JSON | New session / `/resume` |
|
|
170
171
|
| altimate-code | `~/.local/share/altimate-code/opencode.db` | `run -s` / `import` |
|
|
171
172
|
|
|
172
173
|
## Session Summaries
|
|
@@ -261,7 +262,7 @@ A full-screen Textual dashboard with live session browsing, filtering, and analy
|
|
|
261
262
|
│ │ Worked on UTASTAR ensemble │
|
|
262
263
|
│ │ pruning framework... │
|
|
263
264
|
└──────────────────────────┴─────────────────────────────────┘
|
|
264
|
-
Sessions — CC:12 │ Codex:4 │ Alt:3 │ Total:
|
|
265
|
+
Sessions — CC:12 │ Codex:4 │ Gemini:2 │ Alt:3 │ Total:21 │ 129M tok 14:32:01
|
|
265
266
|
```
|
|
266
267
|
|
|
267
268
|
### Key bindings
|
|
@@ -298,7 +299,7 @@ Press `/` to open the filter bar above the session list:
|
|
|
298
299
|
### Transfer modal (`t`)
|
|
299
300
|
|
|
300
301
|
Press `t` on any session to open the transfer panel. Choose:
|
|
301
|
-
- **Target tool** — Claude Code, Codex, or altimate-code
|
|
302
|
+
- **Target tool** — Claude Code, Claude Desktop, Codex, Gemini, or altimate-code
|
|
302
303
|
- **Strategy** — `summary_only`, `key_messages`, or `full_recent`
|
|
303
304
|
- **Preview** — shows the exact shell command (no side effects)
|
|
304
305
|
- **Execute** — builds the bundle and launches the target tool
|
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
**Session manager and context bridge for agentic CLI tools.**
|
|
4
4
|
|
|
5
|
-
ContextForge (`cf`) tracks sessions from Claude Code, Codex, altimate-code, and other
|
|
5
|
+
ContextForge (`cf`) tracks sessions from Claude Code, Claude Desktop, Codex, Gemini, altimate-code, and other
|
|
6
6
|
agentic CLI tools. It generates plain-English summaries of each session and can compact
|
|
7
7
|
and transfer context from one or more sessions into a new session — within the same tool
|
|
8
8
|
or across different tools.
|
|
9
9
|
|
|
10
10
|
## What it does
|
|
11
11
|
|
|
12
|
-
- **Discovers sessions** from Claude Code (`~/.claude/projects/`), Codex (`~/.codex/state_5.sqlite`), and altimate-code (`~/.local/share/altimate-code/`)
|
|
12
|
+
- **Discovers sessions** from Claude Code (`~/.claude/projects/`), Claude Desktop, Codex (`~/.codex/state_5.sqlite`), Gemini (`~/.gemini/tmp/<project_hash>/chats/`), and altimate-code (`~/.local/share/altimate-code/`)
|
|
13
13
|
- **Generates summaries** of what each session accomplished (with optional Claude API integration)
|
|
14
14
|
- **Compacts context** intelligently, reducing multi-turn conversations to a token-efficient ContextBundle
|
|
15
15
|
- **Transfers context** across tools and sessions (Claude Code → Codex, Codex → altimate-code, etc.)
|
|
@@ -19,7 +19,7 @@ or across different tools.
|
|
|
19
19
|
|
|
20
20
|
- **Python 3.13+** (ContextForge requires Python 3.13 or later)
|
|
21
21
|
- **uv** or **pipx** (for installation as a tool)
|
|
22
|
-
- At least one of: Claude Code, Codex, or altimate-code installed and with session history
|
|
22
|
+
- At least one of: Claude Code, Claude Desktop, Codex, Gemini, or altimate-code installed and with session history
|
|
23
23
|
|
|
24
24
|
## Install
|
|
25
25
|
|
|
@@ -139,6 +139,7 @@ The method is chosen automatically based on token count and whether a target ses
|
|
|
139
139
|
| Claude Code (CLI) | `~/.claude/projects/` JSONL | `--system-prompt` / `--resume` |
|
|
140
140
|
| Claude Desktop | `~/Library/Application Support/Claude/local-agent-mode-sessions/` | `--system-prompt` |
|
|
141
141
|
| Codex | `~/.codex/state_5.sqlite` | `resume` / `fork` |
|
|
142
|
+
| Gemini | `~/.gemini/tmp/<project_hash>/chats/` JSON | New session / `/resume` |
|
|
142
143
|
| altimate-code | `~/.local/share/altimate-code/opencode.db` | `run -s` / `import` |
|
|
143
144
|
|
|
144
145
|
## Session Summaries
|
|
@@ -233,7 +234,7 @@ A full-screen Textual dashboard with live session browsing, filtering, and analy
|
|
|
233
234
|
│ │ Worked on UTASTAR ensemble │
|
|
234
235
|
│ │ pruning framework... │
|
|
235
236
|
└──────────────────────────┴─────────────────────────────────┘
|
|
236
|
-
Sessions — CC:12 │ Codex:4 │ Alt:3 │ Total:
|
|
237
|
+
Sessions — CC:12 │ Codex:4 │ Gemini:2 │ Alt:3 │ Total:21 │ 129M tok 14:32:01
|
|
237
238
|
```
|
|
238
239
|
|
|
239
240
|
### Key bindings
|
|
@@ -270,7 +271,7 @@ Press `/` to open the filter bar above the session list:
|
|
|
270
271
|
### Transfer modal (`t`)
|
|
271
272
|
|
|
272
273
|
Press `t` on any session to open the transfer panel. Choose:
|
|
273
|
-
- **Target tool** — Claude Code, Codex, or altimate-code
|
|
274
|
+
- **Target tool** — Claude Code, Claude Desktop, Codex, Gemini, or altimate-code
|
|
274
275
|
- **Strategy** — `summary_only`, `key_messages`, or `full_recent`
|
|
275
276
|
- **Preview** — shows the exact shell command (no side effects)
|
|
276
277
|
- **Execute** — builds the bundle and launches the target tool
|
|
@@ -25,6 +25,16 @@ def _count_tokens(text: str) -> int:
|
|
|
25
25
|
return len(text) // 4
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def _compute_message_tokens(msg) -> int:
|
|
29
|
+
"""Count tokens across all content: text + tool_call inputs + tool_result outputs."""
|
|
30
|
+
total = _count_tokens(msg.content)
|
|
31
|
+
for tc in msg.tool_calls:
|
|
32
|
+
total += _count_tokens(tc.get("input", ""))
|
|
33
|
+
for tr in msg.tool_results:
|
|
34
|
+
total += _count_tokens(tr.get("output", ""))
|
|
35
|
+
return total
|
|
36
|
+
|
|
37
|
+
|
|
28
38
|
class AltimateCodeAdapter(ToolAdapter):
|
|
29
39
|
tool_name = "altimate_code"
|
|
30
40
|
default_paths = [_OPENCODE_DB]
|
|
@@ -86,14 +96,22 @@ class AltimateCodeAdapter(ToolAdapter):
|
|
|
86
96
|
conn = sqlite3.connect(str(_OPENCODE_DB))
|
|
87
97
|
conn.row_factory = sqlite3.Row
|
|
88
98
|
cur = conn.cursor()
|
|
89
|
-
# Join message + part to get full content
|
|
90
99
|
cur.execute(
|
|
91
100
|
"""
|
|
92
|
-
SELECT
|
|
101
|
+
SELECT
|
|
102
|
+
m.id,
|
|
103
|
+
json_extract(m.data, '$.role') as role,
|
|
104
|
+
m.time_created as ts,
|
|
105
|
+
json_extract(p.data, '$.type') as part_type,
|
|
106
|
+
json_extract(p.data, '$.text') as text_content,
|
|
107
|
+
json_extract(p.data, '$.tool') as tool_name,
|
|
108
|
+
json_extract(p.data, '$.callID') as call_id,
|
|
109
|
+
json_extract(p.data, '$.state.input') as tool_input,
|
|
110
|
+
json_extract(p.data, '$.state.output') as tool_output
|
|
93
111
|
FROM message m
|
|
94
112
|
LEFT JOIN part p ON p.message_id = m.id
|
|
95
113
|
WHERE m.session_id = ?
|
|
96
|
-
ORDER BY m.
|
|
114
|
+
ORDER BY m.time_created ASC, p.id ASC
|
|
97
115
|
""",
|
|
98
116
|
(session_id,),
|
|
99
117
|
)
|
|
@@ -103,33 +121,43 @@ class AltimateCodeAdapter(ToolAdapter):
|
|
|
103
121
|
return []
|
|
104
122
|
|
|
105
123
|
messages: list[Message] = []
|
|
124
|
+
current_msg_id: str | None = None
|
|
106
125
|
current_role: str | None = None
|
|
107
126
|
current_parts: list[str] = []
|
|
127
|
+
current_tool_calls: list[dict] = []
|
|
128
|
+
current_tool_results: list[dict] = []
|
|
108
129
|
current_ts: datetime | None = None
|
|
109
130
|
|
|
110
131
|
def flush():
|
|
111
|
-
nonlocal current_role, current_parts, current_ts
|
|
112
|
-
|
|
132
|
+
nonlocal current_msg_id, current_role, current_parts, current_ts
|
|
133
|
+
nonlocal current_tool_calls, current_tool_results
|
|
134
|
+
if current_role and (current_parts or current_tool_calls):
|
|
113
135
|
content = "\n".join(p for p in current_parts if p)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
136
|
+
msg = Message(
|
|
137
|
+
role=current_role,
|
|
138
|
+
content=content,
|
|
139
|
+
timestamp=current_ts,
|
|
140
|
+
tool_calls=list(current_tool_calls),
|
|
141
|
+
tool_results=list(current_tool_results),
|
|
142
|
+
)
|
|
143
|
+
msg.token_count = _compute_message_tokens(msg)
|
|
144
|
+
messages.append(msg)
|
|
145
|
+
current_msg_id = None
|
|
122
146
|
current_role = None
|
|
123
147
|
current_parts = []
|
|
148
|
+
current_tool_calls = []
|
|
149
|
+
current_tool_results = []
|
|
124
150
|
current_ts = None
|
|
125
151
|
|
|
126
152
|
for row in rows:
|
|
153
|
+
msg_id = row["id"]
|
|
127
154
|
role = row["role"]
|
|
128
155
|
if role not in ("user", "assistant"):
|
|
129
156
|
continue
|
|
130
157
|
|
|
131
|
-
if
|
|
158
|
+
if msg_id != current_msg_id:
|
|
132
159
|
flush()
|
|
160
|
+
current_msg_id = msg_id
|
|
133
161
|
current_role = role
|
|
134
162
|
try:
|
|
135
163
|
current_ts = datetime.fromtimestamp(row["ts"] / 1000, tz=timezone.utc)
|
|
@@ -137,28 +165,36 @@ class AltimateCodeAdapter(ToolAdapter):
|
|
|
137
165
|
current_ts = None
|
|
138
166
|
|
|
139
167
|
part_type = row["part_type"] or ""
|
|
140
|
-
content = row["content"] or ""
|
|
141
168
|
|
|
142
|
-
if part_type in ("text", "reasoning")
|
|
143
|
-
|
|
169
|
+
if part_type in ("text", "reasoning"):
|
|
170
|
+
content = row["text_content"] or ""
|
|
171
|
+
if isinstance(content, str) and content:
|
|
144
172
|
try:
|
|
145
173
|
parsed = json.loads(content)
|
|
146
174
|
if isinstance(parsed, dict):
|
|
147
175
|
content = parsed.get("text", str(parsed))
|
|
148
176
|
except (json.JSONDecodeError, TypeError):
|
|
149
177
|
pass
|
|
150
|
-
|
|
178
|
+
content = str(content).strip()
|
|
179
|
+
if content:
|
|
180
|
+
current_parts.append(content)
|
|
181
|
+
|
|
182
|
+
elif part_type == "tool":
|
|
183
|
+
tool_name = row["tool_name"] or "?"
|
|
184
|
+
tool_input = row["tool_input"] or ""
|
|
185
|
+
tool_output = row["tool_output"] or ""
|
|
186
|
+
# input from json_extract is already a JSON string; keep as-is for token counting
|
|
187
|
+
current_tool_calls.append({"name": tool_name, "input": tool_input})
|
|
188
|
+
if tool_output:
|
|
189
|
+
current_tool_results.append({"output": tool_output})
|
|
151
190
|
|
|
152
191
|
flush()
|
|
153
192
|
return messages
|
|
154
193
|
|
|
155
194
|
def _count_session_tokens(self, session_id: str) -> int:
|
|
156
|
-
"""Count total tokens in a session."""
|
|
195
|
+
"""Count total tokens in a session (text + tool inputs + tool outputs)."""
|
|
157
196
|
messages = self.load_messages(session_id)
|
|
158
|
-
|
|
159
|
-
for msg in messages:
|
|
160
|
-
total += _count_tokens(msg.content)
|
|
161
|
-
return total
|
|
197
|
+
return sum(msg.token_count or _compute_message_tokens(msg) for msg in messages)
|
|
162
198
|
|
|
163
199
|
def build_inject_command(
|
|
164
200
|
self,
|
|
@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
|
|
7
7
|
from contextforge.models.session import Message, Session
|
|
8
|
+
from contextforge.utils.tokens import count_tokens
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class ToolAdapter(ABC):
|
|
@@ -42,3 +43,17 @@ class ToolAdapter(ABC):
|
|
|
42
43
|
def is_available(self) -> bool:
|
|
43
44
|
"""Return True if this tool is installed and its data paths exist."""
|
|
44
45
|
return any(p.exists() for p in self.default_paths)
|
|
46
|
+
|
|
47
|
+
def _count_session_tokens(self, session_id: str | Path) -> int:
|
|
48
|
+
"""Count total tokens in a session by loading and summing message tokens.
|
|
49
|
+
|
|
50
|
+
This is the default implementation that all adapters can use.
|
|
51
|
+
Adapters may override this with more efficient implementations.
|
|
52
|
+
"""
|
|
53
|
+
if isinstance(session_id, Path):
|
|
54
|
+
session_id = session_id.stem
|
|
55
|
+
messages = self.load_messages(str(session_id))
|
|
56
|
+
total = 0
|
|
57
|
+
for msg in messages:
|
|
58
|
+
total += msg.token_count or count_tokens(msg.content)
|
|
59
|
+
return total
|
|
@@ -64,6 +64,16 @@ def _count_tokens(text: str) -> int:
|
|
|
64
64
|
return len(text) // 4
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
def _compute_message_tokens(msg: Message) -> int:
|
|
68
|
+
"""Count tokens across all content in a message: text + tool_call inputs + tool_result outputs."""
|
|
69
|
+
total = _count_tokens(msg.content)
|
|
70
|
+
for tc in msg.tool_calls:
|
|
71
|
+
total += _count_tokens(tc.get("input", ""))
|
|
72
|
+
for tr in msg.tool_results:
|
|
73
|
+
total += _count_tokens(tr.get("output", ""))
|
|
74
|
+
return total
|
|
75
|
+
|
|
76
|
+
|
|
67
77
|
class ClaudeCodeAdapter(ToolAdapter):
|
|
68
78
|
tool_name = "claude_code"
|
|
69
79
|
default_paths = [_HISTORY_PATH, _PROJECTS_DIR]
|
|
@@ -309,17 +319,6 @@ class ClaudeCodeAdapter(ToolAdapter):
|
|
|
309
319
|
role = msg_data.get("role", entry_type)
|
|
310
320
|
content_raw = msg_data.get("content", "")
|
|
311
321
|
|
|
312
|
-
# Skip entries where content is an array of tool results/tool use
|
|
313
|
-
# (these are Claude's internal messages, not user input or assistant responses)
|
|
314
|
-
if isinstance(content_raw, list):
|
|
315
|
-
# Only include arrays that contain text blocks (actual assistant content)
|
|
316
|
-
if not any(block.get("type") == "text" for block in content_raw if isinstance(block, dict)):
|
|
317
|
-
continue
|
|
318
|
-
|
|
319
|
-
content = _parse_content(content_raw)
|
|
320
|
-
if not content:
|
|
321
|
-
continue
|
|
322
|
-
|
|
323
322
|
ts_str = entry.get("timestamp")
|
|
324
323
|
ts = None
|
|
325
324
|
if ts_str:
|
|
@@ -328,47 +327,80 @@ class ClaudeCodeAdapter(ToolAdapter):
|
|
|
328
327
|
except (ValueError, AttributeError):
|
|
329
328
|
pass
|
|
330
329
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
330
|
+
if isinstance(content_raw, list):
|
|
331
|
+
has_text = any(
|
|
332
|
+
isinstance(b, dict) and b.get("type") == "text"
|
|
333
|
+
for b in content_raw
|
|
334
|
+
)
|
|
335
|
+
has_tool_result = any(
|
|
336
|
+
isinstance(b, dict) and b.get("type") == "tool_result"
|
|
337
|
+
for b in content_raw
|
|
336
338
|
)
|
|
337
|
-
)
|
|
338
|
-
|
|
339
|
-
return messages
|
|
340
339
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
340
|
+
# tool_result-only user entries: attribute their outputs to the
|
|
341
|
+
# last assistant turn (the one that issued the tool calls)
|
|
342
|
+
if has_tool_result and not has_text:
|
|
343
|
+
if messages and messages[-1].role == "assistant":
|
|
344
|
+
last = messages[-1]
|
|
345
|
+
for block in content_raw:
|
|
346
|
+
if not isinstance(block, dict) or block.get("type") != "tool_result":
|
|
347
|
+
continue
|
|
348
|
+
inner = block.get("content", "")
|
|
349
|
+
if isinstance(inner, list):
|
|
350
|
+
for item in inner:
|
|
351
|
+
if isinstance(item, dict) and item.get("type") == "text":
|
|
352
|
+
output = item.get("text", "")
|
|
353
|
+
if output:
|
|
354
|
+
last.tool_results.append({"output": output})
|
|
355
|
+
elif isinstance(inner, str) and inner:
|
|
356
|
+
last.tool_results.append({"output": inner})
|
|
357
|
+
# Recompute token_count now that tool_results have been added
|
|
358
|
+
last.token_count = _compute_message_tokens(last)
|
|
353
359
|
continue
|
|
354
360
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
361
|
+
# Extract tool_call entries from assistant content arrays
|
|
362
|
+
tool_calls: list[dict] = []
|
|
363
|
+
if role != "user":
|
|
364
|
+
for block in content_raw:
|
|
365
|
+
if not isinstance(block, dict) or block.get("type") != "tool_use":
|
|
366
|
+
continue
|
|
367
|
+
try:
|
|
368
|
+
input_str = json.dumps(block.get("input", {}))
|
|
369
|
+
except (TypeError, ValueError):
|
|
370
|
+
input_str = ""
|
|
371
|
+
tool_calls.append({"name": block.get("name", "?"), "input": input_str})
|
|
372
|
+
else:
|
|
373
|
+
tool_calls = []
|
|
374
|
+
|
|
375
|
+
content = _parse_content(content_raw)
|
|
376
|
+
if not content and not tool_calls:
|
|
377
|
+
continue
|
|
358
378
|
|
|
359
|
-
|
|
360
|
-
|
|
379
|
+
msg = Message(
|
|
380
|
+
role="user" if role == "user" else "assistant",
|
|
381
|
+
content=content,
|
|
382
|
+
timestamp=ts,
|
|
383
|
+
tool_calls=tool_calls,
|
|
384
|
+
)
|
|
385
|
+
msg.token_count = _compute_message_tokens(msg)
|
|
386
|
+
messages.append(msg)
|
|
361
387
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
388
|
+
return messages
|
|
389
|
+
|
|
390
|
+
def _count_session_tokens(self, jsonl_path: Path | str) -> int:
|
|
391
|
+
"""Count total tokens in a session JSONL file or session ID.
|
|
366
392
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
393
|
+
Args:
|
|
394
|
+
jsonl_path: Either a Path to the JSONL file or a session ID string.
|
|
395
|
+
"""
|
|
396
|
+
if isinstance(jsonl_path, str):
|
|
397
|
+
session_id = jsonl_path
|
|
398
|
+
else:
|
|
399
|
+
session_id = jsonl_path.stem
|
|
400
|
+
messages = self.load_messages(session_id)
|
|
401
|
+
total = 0
|
|
402
|
+
for msg in messages:
|
|
403
|
+
total += msg.token_count or _count_tokens(msg.content)
|
|
372
404
|
return total
|
|
373
405
|
|
|
374
406
|
def build_inject_command(
|
|
@@ -240,8 +240,12 @@ class ClaudeDesktopAdapter(ToolAdapter):
|
|
|
240
240
|
pass
|
|
241
241
|
|
|
242
242
|
messages.append(
|
|
243
|
-
Message(
|
|
244
|
-
|
|
243
|
+
Message(
|
|
244
|
+
role="user" if role == "user" else "assistant",
|
|
245
|
+
content=content,
|
|
246
|
+
timestamp=ts,
|
|
247
|
+
token_count=_count_tokens(content),
|
|
248
|
+
)
|
|
245
249
|
)
|
|
246
250
|
return messages
|
|
247
251
|
|
|
@@ -279,7 +283,12 @@ class ClaudeDesktopAdapter(ToolAdapter):
|
|
|
279
283
|
except (ValueError, AttributeError):
|
|
280
284
|
pass
|
|
281
285
|
|
|
282
|
-
messages.append(Message(
|
|
286
|
+
messages.append(Message(
|
|
287
|
+
role=role,
|
|
288
|
+
content=content,
|
|
289
|
+
timestamp=ts,
|
|
290
|
+
token_count=_count_tokens(content),
|
|
291
|
+
))
|
|
283
292
|
return messages
|
|
284
293
|
|
|
285
294
|
def _count_session_tokens(self, session_id: str) -> int:
|
|
@@ -287,7 +296,7 @@ class ClaudeDesktopAdapter(ToolAdapter):
|
|
|
287
296
|
messages = self.load_messages(session_id)
|
|
288
297
|
total = 0
|
|
289
298
|
for msg in messages:
|
|
290
|
-
total += _count_tokens(msg.content)
|
|
299
|
+
total += msg.token_count or _count_tokens(msg.content)
|
|
291
300
|
return total
|
|
292
301
|
|
|
293
302
|
def build_inject_command(
|
|
@@ -7,6 +7,8 @@ import sqlite3
|
|
|
7
7
|
from datetime import datetime, timezone
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
|
|
10
|
+
import tiktoken
|
|
11
|
+
|
|
10
12
|
from contextforge.adapters.base import ToolAdapter
|
|
11
13
|
from contextforge.models.session import Message, Session
|
|
12
14
|
|
|
@@ -29,6 +31,40 @@ _CODEX_SESSIONS_DIR = Path.home() / ".codex" / "sessions"
|
|
|
29
31
|
_SESSION_INDEX = Path.home() / ".codex" / "session_index.jsonl"
|
|
30
32
|
|
|
31
33
|
|
|
34
|
+
def _count_tokens(text: str) -> int:
|
|
35
|
+
"""Count tokens in text using tiktoken for Claude models."""
|
|
36
|
+
try:
|
|
37
|
+
enc = tiktoken.encoding_for_model("claude-3-5-sonnet-20241022")
|
|
38
|
+
return len(enc.encode(text))
|
|
39
|
+
except Exception:
|
|
40
|
+
# Fallback: rough estimate (~4 chars per token)
|
|
41
|
+
return len(text) // 4
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _compute_message_tokens(msg: Message) -> int:
|
|
45
|
+
total = _count_tokens(msg.content)
|
|
46
|
+
for tc in msg.tool_calls:
|
|
47
|
+
total += _count_tokens(tc.get("input", ""))
|
|
48
|
+
for tr in msg.tool_results:
|
|
49
|
+
total += _count_tokens(tr.get("output", ""))
|
|
50
|
+
return total
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _extract_function_output(raw_output: str) -> str:
|
|
54
|
+
"""Extract the human-readable output from a Codex function_call_output payload.
|
|
55
|
+
|
|
56
|
+
The `output` field is a JSON string: {"output": "...", "metadata": {...}}.
|
|
57
|
+
Returns the inner "output" string, or the raw string on parse failure.
|
|
58
|
+
"""
|
|
59
|
+
try:
|
|
60
|
+
parsed = json.loads(raw_output)
|
|
61
|
+
if isinstance(parsed, dict):
|
|
62
|
+
return str(parsed.get("output", raw_output))
|
|
63
|
+
except (json.JSONDecodeError, TypeError):
|
|
64
|
+
pass
|
|
65
|
+
return raw_output
|
|
66
|
+
|
|
67
|
+
|
|
32
68
|
class CodexAdapter(ToolAdapter):
|
|
33
69
|
tool_name = "codex"
|
|
34
70
|
default_paths = [_CODEX_DB, _CODEX_SESSIONS_DIR]
|
|
@@ -75,7 +111,10 @@ class CodexAdapter(ToolAdapter):
|
|
|
75
111
|
cwd=row["cwd"] or None,
|
|
76
112
|
created_at=created,
|
|
77
113
|
updated_at=updated,
|
|
78
|
-
|
|
114
|
+
# tokens_used in state_5.sqlite is cumulative API billing spend
|
|
115
|
+
# (re-sends full context each call), NOT conversation footprint.
|
|
116
|
+
# Leave token_count=None so cf refresh computes it from rollout content.
|
|
117
|
+
token_count=None,
|
|
79
118
|
raw_path=row["rollout_path"] or None,
|
|
80
119
|
status="unknown",
|
|
81
120
|
)
|
|
@@ -172,6 +211,31 @@ class CodexAdapter(ToolAdapter):
|
|
|
172
211
|
|
|
173
212
|
def _parse_rollout(self, path: Path) -> list[Message]:
|
|
174
213
|
messages: list[Message] = []
|
|
214
|
+
|
|
215
|
+
# Pending tool data for the current assistant turn (buffered until
|
|
216
|
+
# we see the agent_message that closes the turn).
|
|
217
|
+
pending_tool_calls: list[dict] = []
|
|
218
|
+
pending_tool_results: list[dict] = []
|
|
219
|
+
|
|
220
|
+
def flush_pending_assistant(text: str = "", ts=None):
|
|
221
|
+
"""Emit an assistant message with any buffered tool data."""
|
|
222
|
+
nonlocal pending_tool_calls, pending_tool_results
|
|
223
|
+
if not text and not pending_tool_calls:
|
|
224
|
+
pending_tool_calls = []
|
|
225
|
+
pending_tool_results = []
|
|
226
|
+
return
|
|
227
|
+
msg = Message(
|
|
228
|
+
role="assistant",
|
|
229
|
+
content=text,
|
|
230
|
+
timestamp=ts,
|
|
231
|
+
tool_calls=list(pending_tool_calls),
|
|
232
|
+
tool_results=list(pending_tool_results),
|
|
233
|
+
)
|
|
234
|
+
msg.token_count = _compute_message_tokens(msg)
|
|
235
|
+
messages.append(msg)
|
|
236
|
+
pending_tool_calls = []
|
|
237
|
+
pending_tool_results = []
|
|
238
|
+
|
|
175
239
|
with path.open() as f:
|
|
176
240
|
for line in f:
|
|
177
241
|
line = line.strip()
|
|
@@ -184,20 +248,42 @@ class CodexAdapter(ToolAdapter):
|
|
|
184
248
|
|
|
185
249
|
entry_type = entry.get("type", "")
|
|
186
250
|
payload = entry.get("payload", {})
|
|
251
|
+
ts_str = entry.get("timestamp")
|
|
252
|
+
ts = None
|
|
253
|
+
if ts_str:
|
|
254
|
+
try:
|
|
255
|
+
ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
|
|
256
|
+
except (ValueError, AttributeError):
|
|
257
|
+
pass
|
|
187
258
|
|
|
188
259
|
if entry_type == "event_msg":
|
|
189
260
|
msg_type = payload.get("type", "")
|
|
190
|
-
# user_message / agent_message are the clean human-facing turns.
|
|
191
|
-
# response_item entries are skipped — they contain system context
|
|
192
|
-
# injections (AGENTS.md, environment_context) and duplicate content.
|
|
193
261
|
if msg_type == "user_message":
|
|
194
|
-
|
|
262
|
+
# New user turn — flush any orphaned tool data first
|
|
263
|
+
flush_pending_assistant()
|
|
264
|
+
text = str(payload.get("message", ""))
|
|
195
265
|
if text:
|
|
196
|
-
|
|
266
|
+
msg = Message(role="user", content=text, timestamp=ts)
|
|
267
|
+
msg.token_count = _count_tokens(text)
|
|
268
|
+
messages.append(msg)
|
|
197
269
|
elif msg_type == "agent_message":
|
|
198
|
-
text = payload.get("message", "")
|
|
199
|
-
|
|
200
|
-
|
|
270
|
+
text = str(payload.get("message", ""))
|
|
271
|
+
flush_pending_assistant(text=text, ts=ts)
|
|
272
|
+
|
|
273
|
+
elif entry_type == "response_item":
|
|
274
|
+
item_type = payload.get("type", "")
|
|
275
|
+
if item_type == "function_call":
|
|
276
|
+
name = payload.get("name", "?")
|
|
277
|
+
arguments = payload.get("arguments", "")
|
|
278
|
+
pending_tool_calls.append({"name": name, "input": arguments})
|
|
279
|
+
elif item_type == "function_call_output":
|
|
280
|
+
raw_output = payload.get("output", "")
|
|
281
|
+
output = _extract_function_output(raw_output)
|
|
282
|
+
if output:
|
|
283
|
+
pending_tool_results.append({"output": output})
|
|
284
|
+
|
|
285
|
+
# Flush any trailing tool data (e.g. incomplete / aborted turn)
|
|
286
|
+
flush_pending_assistant()
|
|
201
287
|
|
|
202
288
|
return messages
|
|
203
289
|
|