lcode-agent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lcode/__init__.py +3 -0
- lcode/__main__.py +3 -0
- lcode/app.py +4140 -0
- lcode/compress.py +80 -0
- lcode/default_config.json +10 -0
- lcode/protocol.py +800 -0
- lcode/settings.py +229 -0
- lcode/store.py +889 -0
- lcode/tools.py +1147 -0
- lcode_agent-0.1.0.dist-info/METADATA +123 -0
- lcode_agent-0.1.0.dist-info/RECORD +13 -0
- lcode_agent-0.1.0.dist-info/WHEEL +4 -0
- lcode_agent-0.1.0.dist-info/entry_points.txt +2 -0
lcode/store.py
ADDED
|
@@ -0,0 +1,889 @@
|
|
|
1
|
+
"""SQLite 会话存储:多轮上下文的真相来源。
|
|
2
|
+
|
|
3
|
+
库文件默认在 ~/.lcode/lcode.db,不进仓库。
|
|
4
|
+
思考只给界面看。发给模型的是摘要 + 近文,含工具调用和结果。
|
|
5
|
+
改文件前打快照,/history 可回退项目。
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import sqlite3
|
|
13
|
+
import time
|
|
14
|
+
import uuid
|
|
15
|
+
from dataclasses import dataclass, replace
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
DB_PATH = Path.home() / ".lcode" / "lcode.db"
|
|
19
|
+
|
|
20
|
+
# 模型窗口未知时的回退;真正阈值 = 窗口 * auto_compact 百分比
|
|
21
|
+
DEFAULT_MODEL_WINDOW = 128_000
|
|
22
|
+
DEFAULT_COMPACT_PERCENT = 85
|
|
23
|
+
SUMMARY_BUDGET = 8_000
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def current_project_path() -> str:
|
|
27
|
+
"""当前工作目录,用来按项目拆开上下文。"""
|
|
28
|
+
return str(Path.cwd().resolve())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _est_tokens(text: str) -> int:
|
|
32
|
+
if not text:
|
|
33
|
+
return 0
|
|
34
|
+
cjk = 0
|
|
35
|
+
other = 0
|
|
36
|
+
for ch in text:
|
|
37
|
+
if ord(ch) >= 0x2E80:
|
|
38
|
+
cjk += 1
|
|
39
|
+
else:
|
|
40
|
+
other += 1
|
|
41
|
+
return cjk + (other + 3) // 4
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
CHECKPOINT_KEEP = 100
|
|
45
|
+
SNAPSHOT_MAX_BYTES = 2_000_000
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class Message:
|
|
50
|
+
id: int
|
|
51
|
+
session_id: str
|
|
52
|
+
role: str
|
|
53
|
+
content: str
|
|
54
|
+
thinking: str
|
|
55
|
+
created_at: int
|
|
56
|
+
project_path: str = ""
|
|
57
|
+
meta: str = ""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Store:
|
|
61
|
+
def __init__(self, path: Path | None = None) -> None:
|
|
62
|
+
self.path = path or DB_PATH
|
|
63
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
64
|
+
self._conn = sqlite3.connect(self.path, timeout=5)
|
|
65
|
+
self._conn.row_factory = sqlite3.Row
|
|
66
|
+
self._conn.execute("PRAGMA journal_mode=WAL")
|
|
67
|
+
self._conn.execute("PRAGMA foreign_keys=ON")
|
|
68
|
+
self._init()
|
|
69
|
+
# 当前会话的内存副本:启动从库载入,之后只追加,请求走这份
|
|
70
|
+
self._session_id: str | None = None
|
|
71
|
+
self._messages: list[Message] = []
|
|
72
|
+
self._summary = ""
|
|
73
|
+
self._summarized_until = 0
|
|
74
|
+
self._project_path = ""
|
|
75
|
+
self.model_window = DEFAULT_MODEL_WINDOW
|
|
76
|
+
self.compact_percent = DEFAULT_COMPACT_PERCENT
|
|
77
|
+
self.compact_at = DEFAULT_MODEL_WINDOW * DEFAULT_COMPACT_PERCENT // 100
|
|
78
|
+
self.context_limit = self.compact_at
|
|
79
|
+
|
|
80
|
+
def set_context_limit(self, tokens: int) -> None:
|
|
81
|
+
self.apply_window(tokens, self.compact_percent)
|
|
82
|
+
|
|
83
|
+
def apply_window(self, context_length: int, percent: int = DEFAULT_COMPACT_PERCENT) -> None:
|
|
84
|
+
"""和 Grok 一样:按模型窗口的百分比自动 compact,磁盘全文仍留着。"""
|
|
85
|
+
self.model_window = max(1024, int(context_length))
|
|
86
|
+
self.compact_percent = min(95, max(50, int(percent)))
|
|
87
|
+
self.compact_at = max(1024, self.model_window * self.compact_percent // 100)
|
|
88
|
+
self.context_limit = self.compact_at
|
|
89
|
+
|
|
90
|
+
def summary_budget(self) -> int:
|
|
91
|
+
cap = max(32, self.compact_at // 8)
|
|
92
|
+
return max(32, min(SUMMARY_BUDGET, cap))
|
|
93
|
+
|
|
94
|
+
def _init(self) -> None:
|
|
95
|
+
self._conn.executescript(
|
|
96
|
+
"""
|
|
97
|
+
CREATE TABLE IF NOT EXISTS session (
|
|
98
|
+
id TEXT PRIMARY KEY,
|
|
99
|
+
created_at INTEGER NOT NULL,
|
|
100
|
+
updated_at INTEGER NOT NULL,
|
|
101
|
+
title TEXT NOT NULL DEFAULT '',
|
|
102
|
+
summary TEXT NOT NULL DEFAULT '',
|
|
103
|
+
summarized_until INTEGER NOT NULL DEFAULT 0,
|
|
104
|
+
project_path TEXT NOT NULL DEFAULT ''
|
|
105
|
+
);
|
|
106
|
+
CREATE TABLE IF NOT EXISTS message (
|
|
107
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
108
|
+
session_id TEXT NOT NULL,
|
|
109
|
+
role TEXT NOT NULL,
|
|
110
|
+
content TEXT NOT NULL,
|
|
111
|
+
thinking TEXT NOT NULL DEFAULT '',
|
|
112
|
+
created_at INTEGER NOT NULL,
|
|
113
|
+
project_path TEXT NOT NULL DEFAULT '',
|
|
114
|
+
FOREIGN KEY (session_id) REFERENCES session(id)
|
|
115
|
+
);
|
|
116
|
+
CREATE INDEX IF NOT EXISTS idx_message_session
|
|
117
|
+
ON message(session_id, id);
|
|
118
|
+
CREATE TABLE IF NOT EXISTS checkpoint (
|
|
119
|
+
id TEXT PRIMARY KEY,
|
|
120
|
+
session_id TEXT NOT NULL,
|
|
121
|
+
message_id INTEGER NOT NULL,
|
|
122
|
+
created_at INTEGER NOT NULL,
|
|
123
|
+
title TEXT NOT NULL DEFAULT '',
|
|
124
|
+
FOREIGN KEY (session_id) REFERENCES session(id)
|
|
125
|
+
);
|
|
126
|
+
CREATE INDEX IF NOT EXISTS idx_checkpoint_session
|
|
127
|
+
ON checkpoint(session_id, message_id);
|
|
128
|
+
CREATE TABLE IF NOT EXISTS snapshot_file (
|
|
129
|
+
checkpoint_id TEXT NOT NULL,
|
|
130
|
+
relpath TEXT NOT NULL,
|
|
131
|
+
existed INTEGER NOT NULL,
|
|
132
|
+
blob_id TEXT,
|
|
133
|
+
PRIMARY KEY (checkpoint_id, relpath)
|
|
134
|
+
);
|
|
135
|
+
CREATE TABLE IF NOT EXISTS snapshot_blob (
|
|
136
|
+
id TEXT PRIMARY KEY,
|
|
137
|
+
content BLOB NOT NULL
|
|
138
|
+
);
|
|
139
|
+
CREATE TABLE IF NOT EXISTS todo (
|
|
140
|
+
project_path TEXT PRIMARY KEY,
|
|
141
|
+
data TEXT NOT NULL DEFAULT '[]'
|
|
142
|
+
);
|
|
143
|
+
"""
|
|
144
|
+
)
|
|
145
|
+
cols = {
|
|
146
|
+
str(row[1])
|
|
147
|
+
for row in self._conn.execute("PRAGMA table_info(session)").fetchall()
|
|
148
|
+
}
|
|
149
|
+
if "summary" not in cols:
|
|
150
|
+
self._conn.execute(
|
|
151
|
+
"ALTER TABLE session ADD COLUMN summary TEXT NOT NULL DEFAULT ''"
|
|
152
|
+
)
|
|
153
|
+
if "summarized_until" not in cols:
|
|
154
|
+
self._conn.execute(
|
|
155
|
+
"ALTER TABLE session ADD COLUMN summarized_until INTEGER NOT NULL DEFAULT 0"
|
|
156
|
+
)
|
|
157
|
+
if "project_path" not in cols:
|
|
158
|
+
self._conn.execute(
|
|
159
|
+
"ALTER TABLE session ADD COLUMN project_path TEXT NOT NULL DEFAULT ''"
|
|
160
|
+
)
|
|
161
|
+
# 每个 session 的 token 记账:重启后底部 ↑↓/cache 接着算,不再归零
|
|
162
|
+
for col in ("tokens_in", "tokens_out", "tokens_cache"):
|
|
163
|
+
if col not in cols:
|
|
164
|
+
self._conn.execute(
|
|
165
|
+
f"ALTER TABLE session ADD COLUMN {col} INTEGER NOT NULL DEFAULT 0"
|
|
166
|
+
)
|
|
167
|
+
msg_cols = {
|
|
168
|
+
str(row[1])
|
|
169
|
+
for row in self._conn.execute("PRAGMA table_info(message)").fetchall()
|
|
170
|
+
}
|
|
171
|
+
if "project_path" not in msg_cols:
|
|
172
|
+
self._conn.execute(
|
|
173
|
+
"ALTER TABLE message ADD COLUMN project_path TEXT NOT NULL DEFAULT ''"
|
|
174
|
+
)
|
|
175
|
+
if "meta" not in msg_cols:
|
|
176
|
+
self._conn.execute(
|
|
177
|
+
"ALTER TABLE message ADD COLUMN meta TEXT NOT NULL DEFAULT ''"
|
|
178
|
+
)
|
|
179
|
+
self._conn.execute(
|
|
180
|
+
"CREATE INDEX IF NOT EXISTS idx_session_project ON session(project_path, updated_at)"
|
|
181
|
+
)
|
|
182
|
+
self._conn.execute(
|
|
183
|
+
"CREATE INDEX IF NOT EXISTS idx_message_project ON message(project_path, id)"
|
|
184
|
+
)
|
|
185
|
+
self._conn.commit()
|
|
186
|
+
|
|
187
|
+
def close(self) -> None:
|
|
188
|
+
self._conn.close()
|
|
189
|
+
|
|
190
|
+
def create_session(self, project_path: str | None = None) -> str:
|
|
191
|
+
sid = uuid.uuid4().hex
|
|
192
|
+
now = int(time.time())
|
|
193
|
+
path = project_path or current_project_path()
|
|
194
|
+
self._conn.execute(
|
|
195
|
+
"""
|
|
196
|
+
INSERT INTO session (id, created_at, updated_at, title, project_path)
|
|
197
|
+
VALUES (?, ?, ?, '', ?)
|
|
198
|
+
""",
|
|
199
|
+
(sid, now, now, path),
|
|
200
|
+
)
|
|
201
|
+
self._conn.commit()
|
|
202
|
+
self._project_path = path
|
|
203
|
+
self._attach(sid, [])
|
|
204
|
+
return sid
|
|
205
|
+
|
|
206
|
+
def latest_session(self) -> str | None:
|
|
207
|
+
row = self._conn.execute(
|
|
208
|
+
"SELECT id FROM session ORDER BY updated_at DESC, created_at DESC LIMIT 1"
|
|
209
|
+
).fetchone()
|
|
210
|
+
return str(row["id"]) if row else None
|
|
211
|
+
|
|
212
|
+
def latest_for_project(self, project_path: str) -> str | None:
|
|
213
|
+
row = self._conn.execute(
|
|
214
|
+
"""
|
|
215
|
+
SELECT id FROM session
|
|
216
|
+
WHERE project_path=?
|
|
217
|
+
ORDER BY updated_at DESC, created_at DESC
|
|
218
|
+
LIMIT 1
|
|
219
|
+
""",
|
|
220
|
+
(project_path,),
|
|
221
|
+
).fetchone()
|
|
222
|
+
return str(row["id"]) if row else None
|
|
223
|
+
|
|
224
|
+
def open_or_create(self, project_path: str | None = None) -> str:
|
|
225
|
+
path = project_path or current_project_path()
|
|
226
|
+
sid = self.latest_for_project(path)
|
|
227
|
+
if not sid:
|
|
228
|
+
# 旧库没有项目路径时,把最近一次空路径会话认领到当前项目
|
|
229
|
+
row = self._conn.execute(
|
|
230
|
+
"""
|
|
231
|
+
SELECT id FROM session
|
|
232
|
+
WHERE project_path='' OR project_path IS NULL
|
|
233
|
+
ORDER BY updated_at DESC, created_at DESC
|
|
234
|
+
LIMIT 1
|
|
235
|
+
"""
|
|
236
|
+
).fetchone()
|
|
237
|
+
if row:
|
|
238
|
+
sid = str(row["id"])
|
|
239
|
+
self._conn.execute(
|
|
240
|
+
"UPDATE session SET project_path=? WHERE id=?",
|
|
241
|
+
(path, sid),
|
|
242
|
+
)
|
|
243
|
+
self._conn.execute(
|
|
244
|
+
"""
|
|
245
|
+
UPDATE message SET project_path=?
|
|
246
|
+
WHERE session_id=? AND (project_path='' OR project_path IS NULL)
|
|
247
|
+
""",
|
|
248
|
+
(path, sid),
|
|
249
|
+
)
|
|
250
|
+
self._conn.commit()
|
|
251
|
+
if sid:
|
|
252
|
+
self._project_path = path
|
|
253
|
+
self._attach(sid, self._fetch_messages(sid))
|
|
254
|
+
return sid
|
|
255
|
+
return self.create_session(path)
|
|
256
|
+
|
|
257
|
+
def list_sessions_for_project(self, project_path: str | None = None) -> list[dict]:
|
|
258
|
+
path = project_path or self._project_path or current_project_path()
|
|
259
|
+
rows = self._conn.execute(
|
|
260
|
+
"""
|
|
261
|
+
SELECT s.id, s.title, s.updated_at,
|
|
262
|
+
(SELECT COUNT(*) FROM message m WHERE m.session_id=s.id) AS n
|
|
263
|
+
FROM session s
|
|
264
|
+
WHERE s.project_path=?
|
|
265
|
+
ORDER BY s.updated_at DESC, s.created_at DESC
|
|
266
|
+
""",
|
|
267
|
+
(path,),
|
|
268
|
+
).fetchall()
|
|
269
|
+
return [
|
|
270
|
+
{
|
|
271
|
+
"id": str(r["id"]),
|
|
272
|
+
"title": str(r["title"] or ""),
|
|
273
|
+
"updated_at": int(r["updated_at"] or 0),
|
|
274
|
+
"n": int(r["n"] or 0),
|
|
275
|
+
}
|
|
276
|
+
for r in rows
|
|
277
|
+
]
|
|
278
|
+
|
|
279
|
+
def switch_session(self, session_id: str) -> str:
|
|
280
|
+
row = self._conn.execute(
|
|
281
|
+
"SELECT id FROM session WHERE id=?", (session_id,)
|
|
282
|
+
).fetchone()
|
|
283
|
+
if not row:
|
|
284
|
+
raise ValueError("窗口不存在")
|
|
285
|
+
self._attach(session_id, self._fetch_messages(session_id))
|
|
286
|
+
return session_id
|
|
287
|
+
|
|
288
|
+
def _attach(self, session_id: str, messages: list[Message]) -> None:
|
|
289
|
+
self._session_id = session_id
|
|
290
|
+
self._messages = list(messages)
|
|
291
|
+
self._summary = ""
|
|
292
|
+
self._summarized_until = 0
|
|
293
|
+
row = self._conn.execute(
|
|
294
|
+
"SELECT summary, summarized_until, project_path FROM session WHERE id=?",
|
|
295
|
+
(session_id,),
|
|
296
|
+
).fetchone()
|
|
297
|
+
if row:
|
|
298
|
+
self._summary = str(row["summary"] or "")
|
|
299
|
+
self._summarized_until = int(row["summarized_until"] or 0)
|
|
300
|
+
path = str(row["project_path"] or "")
|
|
301
|
+
if path:
|
|
302
|
+
self._project_path = path
|
|
303
|
+
|
|
304
|
+
def add_usage(
|
|
305
|
+
self, session_id: str, tokens_in: int, tokens_out: int, tokens_cache: int
|
|
306
|
+
) -> None:
|
|
307
|
+
"""把这一轮的官方用量记到 session 上,重启后还在。"""
|
|
308
|
+
if not session_id or not (tokens_in or tokens_out or tokens_cache):
|
|
309
|
+
return
|
|
310
|
+
self._conn.execute(
|
|
311
|
+
"""
|
|
312
|
+
UPDATE session
|
|
313
|
+
SET tokens_in=tokens_in+?, tokens_out=tokens_out+?,
|
|
314
|
+
tokens_cache=tokens_cache+?
|
|
315
|
+
WHERE id=?
|
|
316
|
+
""",
|
|
317
|
+
(
|
|
318
|
+
max(0, int(tokens_in)),
|
|
319
|
+
max(0, int(tokens_out)),
|
|
320
|
+
max(0, int(tokens_cache)),
|
|
321
|
+
session_id,
|
|
322
|
+
),
|
|
323
|
+
)
|
|
324
|
+
self._conn.commit()
|
|
325
|
+
|
|
326
|
+
def usage_of(self, session_id: str) -> tuple[int, int, int]:
|
|
327
|
+
"""(tokens_in, tokens_out, tokens_cache),没有就全 0。"""
|
|
328
|
+
if not session_id:
|
|
329
|
+
return 0, 0, 0
|
|
330
|
+
row = self._conn.execute(
|
|
331
|
+
"SELECT tokens_in, tokens_out, tokens_cache FROM session WHERE id=?",
|
|
332
|
+
(session_id,),
|
|
333
|
+
).fetchone()
|
|
334
|
+
if not row:
|
|
335
|
+
return 0, 0, 0
|
|
336
|
+
return (
|
|
337
|
+
int(row["tokens_in"] or 0),
|
|
338
|
+
int(row["tokens_out"] or 0),
|
|
339
|
+
int(row["tokens_cache"] or 0),
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
def rename_session(self, session_id: str, title: str) -> None:
|
|
343
|
+
title = " ".join((title or "").strip().split())[:40]
|
|
344
|
+
if not title:
|
|
345
|
+
return
|
|
346
|
+
self._conn.execute(
|
|
347
|
+
"UPDATE session SET title=?, updated_at=? WHERE id=?",
|
|
348
|
+
(title, int(time.time()), session_id),
|
|
349
|
+
)
|
|
350
|
+
self._conn.commit()
|
|
351
|
+
|
|
352
|
+
def delete_session(self, session_id: str) -> None:
|
|
353
|
+
"""删掉整个窗口:消息、检查点、快照文件一起清,blob 引用计数回收。"""
|
|
354
|
+
rows = self._conn.execute(
|
|
355
|
+
"SELECT id FROM checkpoint WHERE session_id=?", (session_id,)
|
|
356
|
+
).fetchall()
|
|
357
|
+
for row in rows:
|
|
358
|
+
self._conn.execute(
|
|
359
|
+
"DELETE FROM snapshot_file WHERE checkpoint_id=?", (str(row["id"]),)
|
|
360
|
+
)
|
|
361
|
+
self._conn.execute("DELETE FROM checkpoint WHERE session_id=?", (session_id,))
|
|
362
|
+
self._conn.execute("DELETE FROM message WHERE session_id=?", (session_id,))
|
|
363
|
+
self._conn.execute("DELETE FROM session WHERE id=?", (session_id,))
|
|
364
|
+
self._gc_blobs()
|
|
365
|
+
self._conn.commit()
|
|
366
|
+
if session_id == self._session_id:
|
|
367
|
+
self._session_id = None
|
|
368
|
+
self._messages = []
|
|
369
|
+
self._summary = ""
|
|
370
|
+
self._summarized_until = 0
|
|
371
|
+
|
|
372
|
+
def save_todos(self, project_path: str, todos: list) -> None:
|
|
373
|
+
data = json.dumps(todos, ensure_ascii=False)
|
|
374
|
+
self._conn.execute(
|
|
375
|
+
"INSERT OR REPLACE INTO todo (project_path, data) VALUES (?, ?)",
|
|
376
|
+
(project_path, data),
|
|
377
|
+
)
|
|
378
|
+
self._conn.commit()
|
|
379
|
+
|
|
380
|
+
def load_todos(self, project_path: str) -> list:
|
|
381
|
+
row = self._conn.execute(
|
|
382
|
+
"SELECT data FROM todo WHERE project_path=?", (project_path,)
|
|
383
|
+
).fetchone()
|
|
384
|
+
if not row:
|
|
385
|
+
return []
|
|
386
|
+
try:
|
|
387
|
+
data = json.loads(str(row["data"] or "[]"))
|
|
388
|
+
except (TypeError, json.JSONDecodeError):
|
|
389
|
+
return []
|
|
390
|
+
return data if isinstance(data, list) else []
|
|
391
|
+
|
|
392
|
+
def touch(self, session_id: str) -> None:
|
|
393
|
+
self._conn.execute(
|
|
394
|
+
"UPDATE session SET updated_at=? WHERE id=?",
|
|
395
|
+
(int(time.time()), session_id),
|
|
396
|
+
)
|
|
397
|
+
self._conn.commit()
|
|
398
|
+
|
|
399
|
+
def set_title_if_empty(self, session_id: str, text: str) -> None:
|
|
400
|
+
title = " ".join(text.strip().split())[:40]
|
|
401
|
+
if not title:
|
|
402
|
+
return
|
|
403
|
+
self._conn.execute(
|
|
404
|
+
"UPDATE session SET title=? WHERE id=? AND title=''",
|
|
405
|
+
(title, session_id),
|
|
406
|
+
)
|
|
407
|
+
self._conn.commit()
|
|
408
|
+
|
|
409
|
+
def add_message(
|
|
410
|
+
self,
|
|
411
|
+
session_id: str,
|
|
412
|
+
role: str,
|
|
413
|
+
content: str,
|
|
414
|
+
thinking: str = "",
|
|
415
|
+
meta: str | dict | None = "",
|
|
416
|
+
) -> int:
|
|
417
|
+
now = int(time.time())
|
|
418
|
+
path = self._project_path or current_project_path()
|
|
419
|
+
if isinstance(meta, dict):
|
|
420
|
+
packed = json.dumps(meta, ensure_ascii=False)
|
|
421
|
+
else:
|
|
422
|
+
packed = str(meta or "")
|
|
423
|
+
cur = self._conn.execute(
|
|
424
|
+
"""
|
|
425
|
+
INSERT INTO message
|
|
426
|
+
(session_id, role, content, thinking, created_at, project_path, meta)
|
|
427
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
428
|
+
""",
|
|
429
|
+
(session_id, role, content, thinking, now, path, packed),
|
|
430
|
+
)
|
|
431
|
+
self._conn.execute(
|
|
432
|
+
"UPDATE session SET updated_at=? WHERE id=?",
|
|
433
|
+
(now, session_id),
|
|
434
|
+
)
|
|
435
|
+
self._conn.commit()
|
|
436
|
+
mid = int(cur.lastrowid)
|
|
437
|
+
msg = Message(
|
|
438
|
+
id=mid,
|
|
439
|
+
session_id=session_id,
|
|
440
|
+
role=role,
|
|
441
|
+
content=content,
|
|
442
|
+
thinking=thinking,
|
|
443
|
+
created_at=now,
|
|
444
|
+
project_path=path,
|
|
445
|
+
meta=packed,
|
|
446
|
+
)
|
|
447
|
+
if session_id == self._session_id:
|
|
448
|
+
self._messages.append(msg)
|
|
449
|
+
return mid
|
|
450
|
+
|
|
451
|
+
def _fetch_messages(self, session_id: str) -> list[Message]:
|
|
452
|
+
rows = self._conn.execute(
|
|
453
|
+
"""
|
|
454
|
+
SELECT id, session_id, role, content, thinking, created_at, project_path, meta
|
|
455
|
+
FROM message
|
|
456
|
+
WHERE session_id=?
|
|
457
|
+
ORDER BY id
|
|
458
|
+
""",
|
|
459
|
+
(session_id,),
|
|
460
|
+
).fetchall()
|
|
461
|
+
return [
|
|
462
|
+
Message(
|
|
463
|
+
id=int(r["id"]),
|
|
464
|
+
session_id=str(r["session_id"]),
|
|
465
|
+
role=str(r["role"]),
|
|
466
|
+
content=str(r["content"]),
|
|
467
|
+
thinking=str(r["thinking"]),
|
|
468
|
+
created_at=int(r["created_at"]),
|
|
469
|
+
project_path=str(r["project_path"] or ""),
|
|
470
|
+
meta=str(r["meta"] or ""),
|
|
471
|
+
)
|
|
472
|
+
for r in rows
|
|
473
|
+
]
|
|
474
|
+
|
|
475
|
+
def merge_message_meta(self, session_id: str, message_id: int, patch: dict) -> None:
|
|
476
|
+
"""往已有消息的 meta 里合并几个字段(如附加图片的文件名)。"""
|
|
477
|
+
row = self._conn.execute(
|
|
478
|
+
"SELECT meta FROM message WHERE id=? AND session_id=?",
|
|
479
|
+
(message_id, session_id),
|
|
480
|
+
).fetchone()
|
|
481
|
+
if not row:
|
|
482
|
+
return
|
|
483
|
+
meta = _meta_dict(str(row["meta"] or ""))
|
|
484
|
+
meta.update(patch)
|
|
485
|
+
packed = json.dumps(meta, ensure_ascii=False)
|
|
486
|
+
self._conn.execute(
|
|
487
|
+
"UPDATE message SET meta=? WHERE id=?", (packed, message_id)
|
|
488
|
+
)
|
|
489
|
+
self._conn.commit()
|
|
490
|
+
if session_id == self._session_id:
|
|
491
|
+
for i, msg in enumerate(self._messages):
|
|
492
|
+
if msg.id == message_id:
|
|
493
|
+
self._messages[i] = replace(msg, meta=packed)
|
|
494
|
+
break
|
|
495
|
+
|
|
496
|
+
def list_messages(self, session_id: str) -> list[Message]:
|
|
497
|
+
if session_id == self._session_id:
|
|
498
|
+
return list(self._messages)
|
|
499
|
+
return self._fetch_messages(session_id)
|
|
500
|
+
|
|
501
|
+
def live_messages(self, session_id: str) -> list[Message]:
|
|
502
|
+
"""还没折进摘要、按当前项目路径注入的原文。"""
|
|
503
|
+
until = self._summarized_until if session_id == self._session_id else self._until_of(session_id)
|
|
504
|
+
path = self._project_path
|
|
505
|
+
rows = self.list_messages(session_id)
|
|
506
|
+
if path:
|
|
507
|
+
rows = [
|
|
508
|
+
m
|
|
509
|
+
for m in rows
|
|
510
|
+
if m.project_path == path or not m.project_path
|
|
511
|
+
]
|
|
512
|
+
return [
|
|
513
|
+
m
|
|
514
|
+
for m in rows
|
|
515
|
+
if m.id > until
|
|
516
|
+
and m.role in ("user", "assistant", "tool")
|
|
517
|
+
and (m.content.strip() or _meta_dict(m.meta).get("tool_calls"))
|
|
518
|
+
]
|
|
519
|
+
|
|
520
|
+
def _until_of(self, session_id: str) -> int:
|
|
521
|
+
if session_id == self._session_id:
|
|
522
|
+
return self._summarized_until
|
|
523
|
+
row = self._conn.execute(
|
|
524
|
+
"SELECT summarized_until FROM session WHERE id=?",
|
|
525
|
+
(session_id,),
|
|
526
|
+
).fetchone()
|
|
527
|
+
return int(row["summarized_until"] or 0) if row else 0
|
|
528
|
+
|
|
529
|
+
def _summary_of(self, session_id: str) -> str:
|
|
530
|
+
if session_id == self._session_id:
|
|
531
|
+
return self._summary
|
|
532
|
+
row = self._conn.execute(
|
|
533
|
+
"SELECT summary FROM session WHERE id=?",
|
|
534
|
+
(session_id,),
|
|
535
|
+
).fetchone()
|
|
536
|
+
return str(row["summary"] or "") if row else ""
|
|
537
|
+
|
|
538
|
+
def summary_messages(self, session_id: str) -> list[dict[str, str]]:
|
|
539
|
+
text = self._summary_of(session_id).strip()
|
|
540
|
+
if not text:
|
|
541
|
+
return []
|
|
542
|
+
return [
|
|
543
|
+
{"role": "user", "content": "【此前对话摘要】\n" + text},
|
|
544
|
+
{"role": "assistant", "content": "已记住摘要,会在此基础上继续。"},
|
|
545
|
+
]
|
|
546
|
+
|
|
547
|
+
def context_for_model(self, session_id: str) -> list[dict]:
|
|
548
|
+
"""给接口用:摘要 + 未折进摘要的原文(含工具)。磁盘全文仍在,这里只组请求。"""
|
|
549
|
+
msgs: list[dict] = self.summary_messages(session_id)
|
|
550
|
+
for row in self.live_messages(session_id):
|
|
551
|
+
item = _message_to_payload(row)
|
|
552
|
+
if item:
|
|
553
|
+
msgs.append(item)
|
|
554
|
+
return msgs
|
|
555
|
+
|
|
556
|
+
def context_tokens(self, session_id: str) -> int:
|
|
557
|
+
return tokens_of(self.context_for_model(session_id))
|
|
558
|
+
|
|
559
|
+
def needs_compress(self, session_id: str) -> bool:
|
|
560
|
+
return self.context_tokens(session_id) > self.context_limit
|
|
561
|
+
|
|
562
|
+
def split_overflow(
|
|
563
|
+
self, session_id: str, *, force: bool = False
|
|
564
|
+
) -> tuple[list[Message], list[Message]]:
|
|
565
|
+
"""超窗时:前面折进摘要,尾巴尽量留到 compact 阈值。force 时至少压掉尾部以外的旧文。"""
|
|
566
|
+
live = self.live_messages(session_id)
|
|
567
|
+
if not live:
|
|
568
|
+
return [], []
|
|
569
|
+
if force and len(live) > 2:
|
|
570
|
+
prefix, tail = live[:-2], live[-2:]
|
|
571
|
+
return _align_tool_tail(live, prefix, tail)
|
|
572
|
+
tail_budget = self.context_limit - self.summary_budget()
|
|
573
|
+
if tail_budget < 32:
|
|
574
|
+
tail_budget = max(32, self.context_limit // 2)
|
|
575
|
+
tail: list[Message] = []
|
|
576
|
+
used = 0
|
|
577
|
+
for msg in reversed(live):
|
|
578
|
+
cost = _est_tokens(msg.content)
|
|
579
|
+
if tail and used + cost > tail_budget:
|
|
580
|
+
break
|
|
581
|
+
tail.append(msg)
|
|
582
|
+
used += cost
|
|
583
|
+
tail.reverse()
|
|
584
|
+
if not tail:
|
|
585
|
+
tail = live[-1:]
|
|
586
|
+
keep = {m.id for m in tail}
|
|
587
|
+
prefix = [m for m in live if m.id not in keep]
|
|
588
|
+
return _align_tool_tail(live, prefix, tail)
|
|
589
|
+
|
|
590
|
+
def apply_summary(self, session_id: str, summary: str, until_id: int) -> None:
|
|
591
|
+
now = int(time.time())
|
|
592
|
+
self._conn.execute(
|
|
593
|
+
"""
|
|
594
|
+
UPDATE session
|
|
595
|
+
SET summary=?, summarized_until=?, updated_at=?
|
|
596
|
+
WHERE id=?
|
|
597
|
+
""",
|
|
598
|
+
(summary, until_id, now, session_id),
|
|
599
|
+
)
|
|
600
|
+
self._conn.commit()
|
|
601
|
+
if session_id == self._session_id:
|
|
602
|
+
self._summary = summary
|
|
603
|
+
self._summarized_until = until_id
|
|
604
|
+
|
|
605
|
+
def _root(self) -> Path:
|
|
606
|
+
return Path(self._project_path or current_project_path()).resolve()
|
|
607
|
+
|
|
608
|
+
def open_checkpoint(self, session_id: str, message_id: int, title: str) -> str:
|
|
609
|
+
cid = uuid.uuid4().hex
|
|
610
|
+
now = int(time.time())
|
|
611
|
+
label = " ".join((title or "").strip().split())[:40]
|
|
612
|
+
self._conn.execute(
|
|
613
|
+
"""
|
|
614
|
+
INSERT INTO checkpoint (id, session_id, message_id, created_at, title)
|
|
615
|
+
VALUES (?, ?, ?, ?, ?)
|
|
616
|
+
""",
|
|
617
|
+
(cid, session_id, message_id, now, label),
|
|
618
|
+
)
|
|
619
|
+
self._conn.commit()
|
|
620
|
+
self._prune_checkpoints(session_id)
|
|
621
|
+
return cid
|
|
622
|
+
|
|
623
|
+
def snapshot_file(self, checkpoint_id: str, path: Path) -> None:
|
|
624
|
+
if not checkpoint_id:
|
|
625
|
+
return
|
|
626
|
+
root = self._root()
|
|
627
|
+
resolved = path.resolve()
|
|
628
|
+
try:
|
|
629
|
+
rel = str(resolved.relative_to(root)).replace("\\", "/")
|
|
630
|
+
except ValueError:
|
|
631
|
+
rel = str(resolved)
|
|
632
|
+
hit = self._conn.execute(
|
|
633
|
+
"""
|
|
634
|
+
SELECT 1 FROM snapshot_file
|
|
635
|
+
WHERE checkpoint_id=? AND relpath=?
|
|
636
|
+
""",
|
|
637
|
+
(checkpoint_id, rel),
|
|
638
|
+
).fetchone()
|
|
639
|
+
if hit:
|
|
640
|
+
return
|
|
641
|
+
existed = 1 if resolved.is_file() else 0
|
|
642
|
+
blob_id = None
|
|
643
|
+
if existed:
|
|
644
|
+
try:
|
|
645
|
+
data = resolved.read_bytes()
|
|
646
|
+
except OSError:
|
|
647
|
+
return
|
|
648
|
+
if len(data) > SNAPSHOT_MAX_BYTES:
|
|
649
|
+
return
|
|
650
|
+
blob_id = hashlib.sha256(data).hexdigest()
|
|
651
|
+
self._conn.execute(
|
|
652
|
+
"INSERT OR IGNORE INTO snapshot_blob (id, content) VALUES (?, ?)",
|
|
653
|
+
(blob_id, data),
|
|
654
|
+
)
|
|
655
|
+
self._conn.execute(
|
|
656
|
+
"""
|
|
657
|
+
INSERT INTO snapshot_file (checkpoint_id, relpath, existed, blob_id)
|
|
658
|
+
VALUES (?, ?, ?, ?)
|
|
659
|
+
""",
|
|
660
|
+
(checkpoint_id, rel, existed, blob_id),
|
|
661
|
+
)
|
|
662
|
+
self._conn.commit()
|
|
663
|
+
|
|
664
|
+
def list_checkpoints(self, session_id: str) -> list[dict]:
|
|
665
|
+
rows = self._conn.execute(
|
|
666
|
+
"""
|
|
667
|
+
SELECT c.id, c.message_id, c.created_at, c.title,
|
|
668
|
+
(SELECT COUNT(*) FROM snapshot_file s WHERE s.checkpoint_id=c.id) AS n
|
|
669
|
+
FROM checkpoint c
|
|
670
|
+
WHERE c.session_id=?
|
|
671
|
+
ORDER BY c.message_id DESC, c.created_at DESC
|
|
672
|
+
""",
|
|
673
|
+
(session_id,),
|
|
674
|
+
).fetchall()
|
|
675
|
+
out = []
|
|
676
|
+
later_files = 0
|
|
677
|
+
for r in rows:
|
|
678
|
+
n = int(r["n"] or 0)
|
|
679
|
+
item = {
|
|
680
|
+
"id": str(r["id"]),
|
|
681
|
+
"message_id": int(r["message_id"]),
|
|
682
|
+
"created_at": int(r["created_at"] or 0),
|
|
683
|
+
"title": str(r["title"] or ""),
|
|
684
|
+
"n": n,
|
|
685
|
+
"later_files": later_files + n,
|
|
686
|
+
}
|
|
687
|
+
later_files += n
|
|
688
|
+
out.append(item)
|
|
689
|
+
return out
|
|
690
|
+
|
|
691
|
+
def restore_files(self, session_id: str, checkpoint_id: str) -> list[str]:
|
|
692
|
+
target = self._conn.execute(
|
|
693
|
+
"SELECT id, message_id FROM checkpoint WHERE id=? AND session_id=?",
|
|
694
|
+
(checkpoint_id, session_id),
|
|
695
|
+
).fetchone()
|
|
696
|
+
if not target:
|
|
697
|
+
raise ValueError("找不到这个快照")
|
|
698
|
+
cut = int(target["message_id"])
|
|
699
|
+
cps = self._conn.execute(
|
|
700
|
+
"""
|
|
701
|
+
SELECT id FROM checkpoint
|
|
702
|
+
WHERE session_id=? AND message_id>=?
|
|
703
|
+
ORDER BY message_id DESC, created_at DESC
|
|
704
|
+
""",
|
|
705
|
+
(session_id, cut),
|
|
706
|
+
).fetchall()
|
|
707
|
+
changed: list[str] = []
|
|
708
|
+
root = self._root()
|
|
709
|
+
for cp in cps:
|
|
710
|
+
files = self._conn.execute(
|
|
711
|
+
"""
|
|
712
|
+
SELECT relpath, existed, blob_id FROM snapshot_file
|
|
713
|
+
WHERE checkpoint_id=?
|
|
714
|
+
""",
|
|
715
|
+
(str(cp["id"]),),
|
|
716
|
+
).fetchall()
|
|
717
|
+
for f in files:
|
|
718
|
+
rel = str(f["relpath"])
|
|
719
|
+
raw = Path(rel)
|
|
720
|
+
dest = raw.resolve() if raw.is_absolute() else (root / rel).resolve()
|
|
721
|
+
if int(f["existed"] or 0) == 0:
|
|
722
|
+
if dest.is_file():
|
|
723
|
+
dest.unlink()
|
|
724
|
+
changed.append(rel)
|
|
725
|
+
continue
|
|
726
|
+
blob_id = f["blob_id"]
|
|
727
|
+
if not blob_id:
|
|
728
|
+
continue
|
|
729
|
+
blob = self._conn.execute(
|
|
730
|
+
"SELECT content FROM snapshot_blob WHERE id=?",
|
|
731
|
+
(blob_id,),
|
|
732
|
+
).fetchone()
|
|
733
|
+
if not blob:
|
|
734
|
+
continue
|
|
735
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
736
|
+
dest.write_bytes(blob["content"])
|
|
737
|
+
changed.append(rel)
|
|
738
|
+
seen: list[str] = []
|
|
739
|
+
for rel in changed:
|
|
740
|
+
if rel not in seen:
|
|
741
|
+
seen.append(rel)
|
|
742
|
+
return seen
|
|
743
|
+
|
|
744
|
+
def drop_checkpoints_after(
|
|
745
|
+
self, session_id: str, message_id: int, *, include: bool = False
|
|
746
|
+
) -> None:
|
|
747
|
+
if include:
|
|
748
|
+
rows = self._conn.execute(
|
|
749
|
+
"SELECT id FROM checkpoint WHERE session_id=? AND message_id>=?",
|
|
750
|
+
(session_id, message_id),
|
|
751
|
+
).fetchall()
|
|
752
|
+
else:
|
|
753
|
+
rows = self._conn.execute(
|
|
754
|
+
"SELECT id FROM checkpoint WHERE session_id=? AND message_id>?",
|
|
755
|
+
(session_id, message_id),
|
|
756
|
+
).fetchall()
|
|
757
|
+
ids = [str(r["id"]) for r in rows]
|
|
758
|
+
for cid in ids:
|
|
759
|
+
self._conn.execute(
|
|
760
|
+
"DELETE FROM snapshot_file WHERE checkpoint_id=?", (cid,)
|
|
761
|
+
)
|
|
762
|
+
self._conn.execute("DELETE FROM checkpoint WHERE id=?", (cid,))
|
|
763
|
+
self._gc_blobs()
|
|
764
|
+
self._conn.commit()
|
|
765
|
+
|
|
766
|
+
def truncate_from_message(self, session_id: str, message_id: int) -> None:
|
|
767
|
+
self._conn.execute(
|
|
768
|
+
"DELETE FROM message WHERE session_id=? AND id>=?",
|
|
769
|
+
(session_id, message_id),
|
|
770
|
+
)
|
|
771
|
+
until = self._until_of(session_id)
|
|
772
|
+
if until >= message_id:
|
|
773
|
+
self._conn.execute(
|
|
774
|
+
"""
|
|
775
|
+
UPDATE session
|
|
776
|
+
SET summary='', summarized_until=0, updated_at=?
|
|
777
|
+
WHERE id=?
|
|
778
|
+
""",
|
|
779
|
+
(int(time.time()), session_id),
|
|
780
|
+
)
|
|
781
|
+
if session_id == self._session_id:
|
|
782
|
+
self._summary = ""
|
|
783
|
+
self._summarized_until = 0
|
|
784
|
+
self.drop_checkpoints_after(session_id, message_id, include=True)
|
|
785
|
+
if session_id == self._session_id:
|
|
786
|
+
self._messages = [m for m in self._messages if m.id < message_id]
|
|
787
|
+
self._conn.commit()
|
|
788
|
+
|
|
789
|
+
def _prune_checkpoints(self, session_id: str) -> None:
|
|
790
|
+
rows = self._conn.execute(
|
|
791
|
+
"""
|
|
792
|
+
SELECT id FROM checkpoint
|
|
793
|
+
WHERE session_id=?
|
|
794
|
+
ORDER BY message_id DESC, created_at DESC
|
|
795
|
+
""",
|
|
796
|
+
(session_id,),
|
|
797
|
+
).fetchall()
|
|
798
|
+
extra = [str(r["id"]) for r in rows[CHECKPOINT_KEEP:]]
|
|
799
|
+
for cid in extra:
|
|
800
|
+
self._conn.execute(
|
|
801
|
+
"DELETE FROM snapshot_file WHERE checkpoint_id=?", (cid,)
|
|
802
|
+
)
|
|
803
|
+
self._conn.execute("DELETE FROM checkpoint WHERE id=?", (cid,))
|
|
804
|
+
if extra:
|
|
805
|
+
self._gc_blobs()
|
|
806
|
+
self._conn.commit()
|
|
807
|
+
|
|
808
|
+
def _gc_blobs(self) -> None:
|
|
809
|
+
self._conn.execute(
|
|
810
|
+
"""
|
|
811
|
+
DELETE FROM snapshot_blob
|
|
812
|
+
WHERE id NOT IN (
|
|
813
|
+
SELECT blob_id FROM snapshot_file WHERE blob_id IS NOT NULL
|
|
814
|
+
)
|
|
815
|
+
"""
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _align_tool_tail(
|
|
820
|
+
live: list[Message], prefix: list[Message], tail: list[Message]
|
|
821
|
+
) -> tuple[list[Message], list[Message]]:
|
|
822
|
+
"""尾巴不要从孤立的 tool 结果起头,把对应的 assistant tool_calls 拉回来。"""
|
|
823
|
+
if not tail:
|
|
824
|
+
return prefix, tail
|
|
825
|
+
by_id = {m.id: i for i, m in enumerate(live)}
|
|
826
|
+
keep = {m.id for m in tail}
|
|
827
|
+
while tail and tail[0].role == "tool":
|
|
828
|
+
idx = by_id.get(tail[0].id)
|
|
829
|
+
if idx is None or idx <= 0:
|
|
830
|
+
break
|
|
831
|
+
prev = live[idx - 1]
|
|
832
|
+
if prev.id in keep:
|
|
833
|
+
break
|
|
834
|
+
tail.insert(0, prev)
|
|
835
|
+
keep.add(prev.id)
|
|
836
|
+
prefix = [m for m in live if m.id not in keep]
|
|
837
|
+
return prefix, tail
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _meta_dict(raw: str) -> dict:
|
|
841
|
+
if not raw:
|
|
842
|
+
return {}
|
|
843
|
+
try:
|
|
844
|
+
data = json.loads(raw)
|
|
845
|
+
except (TypeError, json.JSONDecodeError):
|
|
846
|
+
return {}
|
|
847
|
+
return data if isinstance(data, dict) else {}
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _message_to_payload(row: Message) -> dict | None:
|
|
851
|
+
meta = _meta_dict(row.meta)
|
|
852
|
+
if row.role == "tool":
|
|
853
|
+
item = {
|
|
854
|
+
"role": "tool",
|
|
855
|
+
"content": row.content or "",
|
|
856
|
+
"tool_call_id": str(meta.get("tool_call_id") or ""),
|
|
857
|
+
}
|
|
858
|
+
name = str(meta.get("name") or "")
|
|
859
|
+
if name:
|
|
860
|
+
item["name"] = name
|
|
861
|
+
return item
|
|
862
|
+
if row.role == "assistant":
|
|
863
|
+
item: dict = {"role": "assistant"}
|
|
864
|
+
text = (row.content or "").strip()
|
|
865
|
+
calls = meta.get("tool_calls")
|
|
866
|
+
if text:
|
|
867
|
+
item["content"] = text
|
|
868
|
+
if calls:
|
|
869
|
+
item["tool_calls"] = calls
|
|
870
|
+
if "content" not in item and not calls:
|
|
871
|
+
return None
|
|
872
|
+
return item
|
|
873
|
+
text = (row.content or "").strip()
|
|
874
|
+
if not text:
|
|
875
|
+
return None
|
|
876
|
+
return {"role": row.role, "content": text}
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def tokens_of(msgs: list[dict]) -> int:
|
|
880
|
+
total = 0
|
|
881
|
+
for msg in msgs:
|
|
882
|
+
total += _est_tokens(str(msg.get("content") or ""))
|
|
883
|
+
calls = msg.get("tool_calls")
|
|
884
|
+
if calls:
|
|
885
|
+
try:
|
|
886
|
+
total += _est_tokens(json.dumps(calls, ensure_ascii=False))
|
|
887
|
+
except (TypeError, ValueError):
|
|
888
|
+
pass
|
|
889
|
+
return total
|