chat-memory-sync 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chat_memory_sync/__init__.py +2 -0
- chat_memory_sync/__main__.py +5 -0
- chat_memory_sync/cli.py +62 -0
- chat_memory_sync/cmlib.py +76 -0
- chat_memory_sync/install.py +221 -0
- chat_memory_sync/recall.py +231 -0
- chat_memory_sync/run.py +92 -0
- chat_memory_sync/sync.py +938 -0
- chat_memory_sync-0.1.3.dist-info/METADATA +204 -0
- chat_memory_sync-0.1.3.dist-info/RECORD +13 -0
- chat_memory_sync-0.1.3.dist-info/WHEEL +4 -0
- chat_memory_sync-0.1.3.dist-info/entry_points.txt +3 -0
- chat_memory_sync-0.1.3.dist-info/licenses/LICENSE +21 -0
chat_memory_sync/sync.py
ADDED
|
@@ -0,0 +1,938 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
chat-memory sync: export chats from multiple AI coding harnesses into one
|
|
4
|
+
project-organized Markdown memory bank.
|
|
5
|
+
|
|
6
|
+
Harnesses: cursor, cursor-agent, claude-code, opencode, pi, codex, gemini.
|
|
7
|
+
|
|
8
|
+
Each chat -> <data_dir>/projects/<project>/<harness>/<date>_<client>_<slug>_<id8>.md
|
|
9
|
+
with YAML frontmatter. Per-project and global INDEX.md are regenerated. Idempotent:
|
|
10
|
+
files are only rewritten when content changes (updates to OLD chats included).
|
|
11
|
+
|
|
12
|
+
Data-repo location + client come from cmlib (config-driven, machine-local state).
|
|
13
|
+
"""
|
|
14
|
+
import json, os, sqlite3, re, glob, datetime, socket, platform
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from . import cmlib
|
|
17
|
+
|
|
18
|
+
HOME = os.path.expanduser("~")
|
|
19
|
+
OUT = cmlib.data_dir()
|
|
20
|
+
PROJ_DIR = cmlib.projects_dir()
|
|
21
|
+
CLIENT = cmlib.client_id()
|
|
22
|
+
PLATFORM = cmlib.PLATFORM
|
|
23
|
+
|
|
24
|
+
def _first_existing(paths):
|
|
25
|
+
for p in paths:
|
|
26
|
+
if p and os.path.exists(p):
|
|
27
|
+
return p
|
|
28
|
+
return paths[0] if paths else ""
|
|
29
|
+
|
|
30
|
+
def cursor_user_dir():
|
|
31
|
+
s = platform.system()
|
|
32
|
+
if s == "Darwin":
|
|
33
|
+
return os.path.join(HOME, "Library/Application Support/Cursor/User")
|
|
34
|
+
if s == "Windows":
|
|
35
|
+
appdata = os.environ.get("APPDATA", os.path.join(HOME, "AppData", "Roaming"))
|
|
36
|
+
return os.path.join(appdata, "Cursor", "User")
|
|
37
|
+
return os.path.join(HOME, ".config/Cursor/User")
|
|
38
|
+
|
|
39
|
+
def opencode_db_path():
|
|
40
|
+
cands = [os.path.join(HOME, ".local/share/opencode/opencode.db")]
|
|
41
|
+
la = os.environ.get("LOCALAPPDATA")
|
|
42
|
+
if la:
|
|
43
|
+
cands.append(os.path.join(la, "opencode", "opencode.db"))
|
|
44
|
+
cands.append(os.path.join(HOME, "Library/Application Support/opencode/opencode.db"))
|
|
45
|
+
return _first_existing(cands)
|
|
46
|
+
|
|
47
|
+
# ---------------------------------------------------------------- helpers ----
|
|
48
|
+
|
|
49
|
+
def toms(x):
|
|
50
|
+
if isinstance(x, (int, float)):
|
|
51
|
+
return int(x)
|
|
52
|
+
if isinstance(x, str):
|
|
53
|
+
try:
|
|
54
|
+
return int(datetime.datetime.fromisoformat(x.replace("Z", "+00:00")).timestamp() * 1000)
|
|
55
|
+
except Exception:
|
|
56
|
+
return 0
|
|
57
|
+
return 0
|
|
58
|
+
|
|
59
|
+
def iso(ms):
|
|
60
|
+
if not ms:
|
|
61
|
+
return ""
|
|
62
|
+
try:
|
|
63
|
+
return datetime.datetime.fromtimestamp(ms / 1000).strftime("%Y-%m-%dT%H:%M:%S")
|
|
64
|
+
except Exception:
|
|
65
|
+
return ""
|
|
66
|
+
|
|
67
|
+
def datestr(ms):
|
|
68
|
+
try:
|
|
69
|
+
return datetime.datetime.fromtimestamp(ms / 1000).strftime("%Y-%m-%d") if ms else "nodate"
|
|
70
|
+
except Exception:
|
|
71
|
+
return "nodate"
|
|
72
|
+
|
|
73
|
+
def slug(s, n=55):
|
|
74
|
+
s = re.sub(r"\s+", " ", (s or "").strip())
|
|
75
|
+
s = re.sub(r"[^\w\- ]", "", s).strip().replace(" ", "-")
|
|
76
|
+
return s[:n] or "untitled"
|
|
77
|
+
|
|
78
|
+
# parents under which the first segment is the project/repo name
|
|
79
|
+
WS_MARKERS = ["/ws/", "/pspace/", "/src/", "/code/", "/projects/", "/Developer/"]
|
|
80
|
+
|
|
81
|
+
def canonical_project(path):
|
|
82
|
+
"""Map any cwd/worktree/subapp path to a single repo-level project key."""
|
|
83
|
+
if not path:
|
|
84
|
+
return "_unknown"
|
|
85
|
+
p = path.rstrip("/")
|
|
86
|
+
for m in WS_MARKERS:
|
|
87
|
+
i = p.find(m)
|
|
88
|
+
if i != -1:
|
|
89
|
+
rest = p[i + len(m):].split("/")
|
|
90
|
+
rest = [x for x in rest if x]
|
|
91
|
+
if not rest:
|
|
92
|
+
continue
|
|
93
|
+
if rest[0] == ".worktrees" and len(rest) > 1:
|
|
94
|
+
return slug(rest[1], 60)
|
|
95
|
+
return slug(rest[0], 60)
|
|
96
|
+
base = os.path.basename(p)
|
|
97
|
+
return slug(base, 60) if base else "_unknown"
|
|
98
|
+
|
|
99
|
+
def render_message(m):
|
|
100
|
+
role = m.get("role", "?")
|
|
101
|
+
label = {"user": "User", "assistant": "Assistant", "system": "System"}.get(role, role.title())
|
|
102
|
+
model = m.get("model") or ""
|
|
103
|
+
hdr = f"### {label}" + (f" _(model: {model})_" if model and role == "assistant" else "")
|
|
104
|
+
body = []
|
|
105
|
+
th = (m.get("thinking") or "").strip()
|
|
106
|
+
if th:
|
|
107
|
+
body.append("<details><summary>thinking</summary>\n\n" + th + "\n\n</details>")
|
|
108
|
+
txt = (m.get("text") or "").strip()
|
|
109
|
+
if txt:
|
|
110
|
+
body.append(txt)
|
|
111
|
+
for t in (m.get("tools") or []):
|
|
112
|
+
nm = t.get("name") or "tool"
|
|
113
|
+
args = t.get("args") or ""
|
|
114
|
+
if not isinstance(args, str):
|
|
115
|
+
args = json.dumps(args, ensure_ascii=False)
|
|
116
|
+
if len(args) > 1200:
|
|
117
|
+
args = args[:1200] + " ...(truncated)"
|
|
118
|
+
block = f"**\u2192 tool: `{nm}`**"
|
|
119
|
+
if args.strip():
|
|
120
|
+
block += f"\n```json\n{args}\n```"
|
|
121
|
+
res = t.get("result")
|
|
122
|
+
if isinstance(res, str) and res.strip():
|
|
123
|
+
r = res if len(res) < 1000 else res[:1000] + " ...(truncated)"
|
|
124
|
+
block += f"\n<details><summary>result</summary>\n\n```\n{r}\n```\n\n</details>"
|
|
125
|
+
body.append(block)
|
|
126
|
+
if not body:
|
|
127
|
+
return None
|
|
128
|
+
return hdr + "\n\n" + "\n\n".join(body)
|
|
129
|
+
|
|
130
|
+
def render_chat(c):
|
|
131
|
+
msgs = [render_message(m) for m in c["messages"]]
|
|
132
|
+
msgs = [m for m in msgs if m]
|
|
133
|
+
if not msgs:
|
|
134
|
+
return None
|
|
135
|
+
models = ", ".join(sorted({m for m in c.get("models", []) if m}))
|
|
136
|
+
title = c["title"] or c["id"][:8]
|
|
137
|
+
fm = [
|
|
138
|
+
"---",
|
|
139
|
+
f"client: {c.get('client', CLIENT)}",
|
|
140
|
+
f"platform: {c.get('platform', PLATFORM)}",
|
|
141
|
+
f"harness: {c['harness']}",
|
|
142
|
+
f"chat_id: {c['id']}",
|
|
143
|
+
f"project: {c['project']}",
|
|
144
|
+
f"source_path: {c.get('source_path','') or ''}",
|
|
145
|
+
f"title: {json.dumps(title, ensure_ascii=False)}",
|
|
146
|
+
f"created: {iso(c['created'])}",
|
|
147
|
+
f"updated: {iso(c['updated'])}",
|
|
148
|
+
f"models: {json.dumps(models)}",
|
|
149
|
+
f"messages: {len(msgs)}",
|
|
150
|
+
"---",
|
|
151
|
+
"",
|
|
152
|
+
f"# {title}",
|
|
153
|
+
"",
|
|
154
|
+
]
|
|
155
|
+
return "\n".join(fm) + "\n\n".join(msgs) + "\n"
|
|
156
|
+
|
|
157
|
+
def write_if_changed(path, content):
|
|
158
|
+
if os.path.isfile(path):
|
|
159
|
+
with open(path) as f:
|
|
160
|
+
if f.read() == content:
|
|
161
|
+
return "unchanged"
|
|
162
|
+
st = "updated"
|
|
163
|
+
else:
|
|
164
|
+
st = "new"
|
|
165
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
166
|
+
with open(path, "w") as f:
|
|
167
|
+
f.write(content)
|
|
168
|
+
return st
|
|
169
|
+
|
|
170
|
+
# --------------------------------------------------------------- CURSOR ------
|
|
171
|
+
|
|
172
|
+
def cursor_chats():
|
|
173
|
+
userdir = cursor_user_dir()
|
|
174
|
+
db = os.path.join(userdir, "globalStorage", "state.vscdb")
|
|
175
|
+
if not os.path.isfile(db):
|
|
176
|
+
return
|
|
177
|
+
# workspace fallback map (composerId -> folder)
|
|
178
|
+
wsmap = {}
|
|
179
|
+
wsroot = os.path.join(userdir, "workspaceStorage")
|
|
180
|
+
for d in glob.glob(os.path.join(wsroot, "*")):
|
|
181
|
+
dbp = os.path.join(d, "state.vscdb")
|
|
182
|
+
if not os.path.isfile(dbp):
|
|
183
|
+
continue
|
|
184
|
+
folder = ""
|
|
185
|
+
j = os.path.join(d, "workspace.json")
|
|
186
|
+
if os.path.isfile(j):
|
|
187
|
+
try:
|
|
188
|
+
folder = json.load(open(j)).get("folder", "").replace("file://", "")
|
|
189
|
+
except Exception:
|
|
190
|
+
pass
|
|
191
|
+
try:
|
|
192
|
+
con = sqlite3.connect(f"file:{dbp}?mode=ro", uri=True)
|
|
193
|
+
r = con.execute("SELECT value FROM ItemTable WHERE key='composer.composerData'").fetchone()
|
|
194
|
+
if r:
|
|
195
|
+
for comp in json.loads(r[0]).get("allComposers", []):
|
|
196
|
+
cid = comp.get("composerId")
|
|
197
|
+
if cid:
|
|
198
|
+
wsmap[cid] = folder
|
|
199
|
+
con.close()
|
|
200
|
+
except Exception:
|
|
201
|
+
pass
|
|
202
|
+
|
|
203
|
+
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
204
|
+
cur = con.cursor()
|
|
205
|
+
fetch = con.cursor()
|
|
206
|
+
bub = defaultdict(list)
|
|
207
|
+
cur.execute("SELECT key FROM cursorDiskKV WHERE key LIKE 'bubbleId:%'")
|
|
208
|
+
for (k,) in cur.fetchall():
|
|
209
|
+
parts = k.split(":", 2)
|
|
210
|
+
if len(parts) == 3:
|
|
211
|
+
bub[parts[1]].append(parts[2])
|
|
212
|
+
|
|
213
|
+
cur.execute("SELECT value FROM cursorDiskKV WHERE key LIKE 'composerData:%'")
|
|
214
|
+
comps = []
|
|
215
|
+
for (v,) in cur.fetchall():
|
|
216
|
+
try:
|
|
217
|
+
comps.append(json.loads(v))
|
|
218
|
+
except Exception:
|
|
219
|
+
pass
|
|
220
|
+
|
|
221
|
+
def get_thinking(b):
|
|
222
|
+
out = []
|
|
223
|
+
th = b.get("thinking")
|
|
224
|
+
if isinstance(th, str) and th.strip():
|
|
225
|
+
try:
|
|
226
|
+
j = json.loads(th)
|
|
227
|
+
out.append(j.get("text") if isinstance(j, dict) else th)
|
|
228
|
+
except Exception:
|
|
229
|
+
out.append(th)
|
|
230
|
+
elif isinstance(th, dict) and th.get("text"):
|
|
231
|
+
out.append(th["text"])
|
|
232
|
+
for t in (b.get("allThinkingBlocks") or []):
|
|
233
|
+
if isinstance(t, dict) and (t.get("text") or t.get("content")):
|
|
234
|
+
out.append(t.get("text") or t.get("content"))
|
|
235
|
+
return "\n\n".join(x.strip() for x in out if x and x.strip())
|
|
236
|
+
|
|
237
|
+
for d in comps:
|
|
238
|
+
cid = d.get("composerId")
|
|
239
|
+
if not cid:
|
|
240
|
+
continue
|
|
241
|
+
headers = d.get("fullConversationHeadersOnly") or []
|
|
242
|
+
created = toms(d.get("createdAt"))
|
|
243
|
+
updated = toms(d.get("lastUpdatedAt")) or created
|
|
244
|
+
src = ""
|
|
245
|
+
wi = d.get("workspaceIdentifier") or {}
|
|
246
|
+
if isinstance(wi, dict):
|
|
247
|
+
uri = wi.get("uri") or {}
|
|
248
|
+
if isinstance(uri, dict):
|
|
249
|
+
src = uri.get("fsPath") or uri.get("path") or ""
|
|
250
|
+
if not src:
|
|
251
|
+
src = wsmap.get(cid, "")
|
|
252
|
+
if headers:
|
|
253
|
+
keys = [f"bubbleId:{cid}:{h.get('bubbleId')}" for h in headers if isinstance(h, dict) and h.get("bubbleId")]
|
|
254
|
+
ordered_keys = keys
|
|
255
|
+
else:
|
|
256
|
+
keys = [f"bubbleId:{cid}:{b}" for b in bub.get(cid, [])]
|
|
257
|
+
ordered_keys = None
|
|
258
|
+
rows = {}
|
|
259
|
+
for i in range(0, len(keys), 400):
|
|
260
|
+
chunk = keys[i:i + 400]
|
|
261
|
+
q = "SELECT key,value FROM cursorDiskKV WHERE key IN (%s)" % ",".join("?" * len(chunk))
|
|
262
|
+
for k, v in fetch.execute(q, chunk).fetchall():
|
|
263
|
+
rows[k] = v
|
|
264
|
+
bubbles = []
|
|
265
|
+
if ordered_keys:
|
|
266
|
+
seq = [rows[k] for k in ordered_keys if k in rows]
|
|
267
|
+
else:
|
|
268
|
+
tmp = []
|
|
269
|
+
for v in rows.values():
|
|
270
|
+
try:
|
|
271
|
+
b = json.loads(v)
|
|
272
|
+
except Exception:
|
|
273
|
+
continue
|
|
274
|
+
tmp.append((b.get("startedAtMs") or toms(b.get("createdAt")), v))
|
|
275
|
+
tmp.sort(key=lambda x: x[0])
|
|
276
|
+
seq = [v for _, v in tmp]
|
|
277
|
+
messages = []
|
|
278
|
+
models = set()
|
|
279
|
+
title = d.get("name") or ""
|
|
280
|
+
for v in seq:
|
|
281
|
+
try:
|
|
282
|
+
b = json.loads(v)
|
|
283
|
+
except Exception:
|
|
284
|
+
continue
|
|
285
|
+
typ = b.get("type")
|
|
286
|
+
role = "user" if typ == 1 else "assistant" if typ == 2 else "other"
|
|
287
|
+
mi = b.get("modelInfo") or {}
|
|
288
|
+
model = mi.get("modelName") or mi.get("model") or "" if isinstance(mi, dict) else ""
|
|
289
|
+
if model:
|
|
290
|
+
models.add(model)
|
|
291
|
+
tools = []
|
|
292
|
+
tf = b.get("toolFormerData")
|
|
293
|
+
if isinstance(tf, dict) and tf.get("name"):
|
|
294
|
+
tools.append({"name": tf["name"], "args": tf.get("rawArgs") or "", "result": tf.get("result")})
|
|
295
|
+
msg = {"role": role, "model": model, "thinking": get_thinking(b),
|
|
296
|
+
"text": b.get("text") or "", "tools": tools}
|
|
297
|
+
messages.append(msg)
|
|
298
|
+
if not title and role == "user" and (b.get("text") or "").strip():
|
|
299
|
+
title = b["text"].strip().splitlines()[0][:80]
|
|
300
|
+
if not messages:
|
|
301
|
+
continue
|
|
302
|
+
yield {"harness": "cursor", "id": cid, "project": canonical_project(src),
|
|
303
|
+
"source_path": src, "title": title, "created": created,
|
|
304
|
+
"updated": updated, "models": models, "messages": messages}
|
|
305
|
+
|
|
306
|
+
# ------------------------------------------------------------ CLAUDE CODE ----
|
|
307
|
+
|
|
308
|
+
def claude_code_chats():
|
|
309
|
+
root = os.path.join(HOME, ".claude/projects")
|
|
310
|
+
for f in glob.glob(os.path.join(root, "*", "*.jsonl")):
|
|
311
|
+
sid = os.path.splitext(os.path.basename(f))[0]
|
|
312
|
+
cwd = ""
|
|
313
|
+
created = updated = 0
|
|
314
|
+
title = ""
|
|
315
|
+
messages = []
|
|
316
|
+
models = set()
|
|
317
|
+
try:
|
|
318
|
+
lines = open(f, encoding="utf-8").read().splitlines()
|
|
319
|
+
except Exception:
|
|
320
|
+
continue
|
|
321
|
+
for line in lines:
|
|
322
|
+
if not line.strip():
|
|
323
|
+
continue
|
|
324
|
+
try:
|
|
325
|
+
d = json.loads(line)
|
|
326
|
+
except Exception:
|
|
327
|
+
continue
|
|
328
|
+
t = d.get("type")
|
|
329
|
+
if d.get("cwd"):
|
|
330
|
+
cwd = d["cwd"]
|
|
331
|
+
tms = toms(d.get("timestamp"))
|
|
332
|
+
if tms:
|
|
333
|
+
created = tms if not created else min(created, tms)
|
|
334
|
+
updated = max(updated, tms)
|
|
335
|
+
if t == "summary" and d.get("summary"):
|
|
336
|
+
title = d["summary"]
|
|
337
|
+
continue
|
|
338
|
+
if t not in ("user", "assistant"):
|
|
339
|
+
continue
|
|
340
|
+
m = d.get("message") or {}
|
|
341
|
+
role = m.get("role") or t
|
|
342
|
+
model = m.get("model") or ""
|
|
343
|
+
if model and model != "<synthetic>":
|
|
344
|
+
models.add(model)
|
|
345
|
+
else:
|
|
346
|
+
model = ""
|
|
347
|
+
content = m.get("content")
|
|
348
|
+
text_parts, think, tools = [], [], []
|
|
349
|
+
if isinstance(content, str):
|
|
350
|
+
text_parts.append(content)
|
|
351
|
+
elif isinstance(content, list):
|
|
352
|
+
for blk in content:
|
|
353
|
+
if not isinstance(blk, dict):
|
|
354
|
+
continue
|
|
355
|
+
bt = blk.get("type")
|
|
356
|
+
if bt == "text":
|
|
357
|
+
text_parts.append(blk.get("text", ""))
|
|
358
|
+
elif bt == "thinking":
|
|
359
|
+
think.append(blk.get("thinking", "") or blk.get("text", ""))
|
|
360
|
+
elif bt == "tool_use":
|
|
361
|
+
tools.append({"name": blk.get("name"), "args": blk.get("input"), "result": None})
|
|
362
|
+
elif bt == "tool_result":
|
|
363
|
+
rc = blk.get("content")
|
|
364
|
+
if isinstance(rc, list):
|
|
365
|
+
rc = " ".join(x.get("text", "") for x in rc if isinstance(x, dict))
|
|
366
|
+
tools.append({"name": "result", "args": "", "result": rc if isinstance(rc, str) else json.dumps(rc)[:1000]})
|
|
367
|
+
msg = {"role": role, "model": model, "thinking": "\n\n".join(think),
|
|
368
|
+
"text": "\n".join(tp for tp in text_parts if tp).strip(), "tools": tools}
|
|
369
|
+
if msg["text"] or msg["thinking"] or msg["tools"]:
|
|
370
|
+
messages.append(msg)
|
|
371
|
+
if not title and role == "user" and msg["text"]:
|
|
372
|
+
title = msg["text"].splitlines()[0][:80]
|
|
373
|
+
if not messages:
|
|
374
|
+
continue
|
|
375
|
+
yield {"harness": "claude-code", "id": sid, "project": canonical_project(cwd),
|
|
376
|
+
"source_path": cwd, "title": title, "created": created,
|
|
377
|
+
"updated": updated, "models": models, "messages": messages}
|
|
378
|
+
|
|
379
|
+
# ------------------------------------------------------------------- PI ------
|
|
380
|
+
|
|
381
|
+
def pi_chats():
|
|
382
|
+
root = os.path.join(HOME, ".pi/agent/sessions")
|
|
383
|
+
for f in glob.glob(os.path.join(root, "*", "*.jsonl")):
|
|
384
|
+
cwd = ""
|
|
385
|
+
created = updated = 0
|
|
386
|
+
sid = os.path.splitext(os.path.basename(f))[0].split("_", 1)[-1]
|
|
387
|
+
title = ""
|
|
388
|
+
messages = []
|
|
389
|
+
models = set()
|
|
390
|
+
cur_model = ""
|
|
391
|
+
try:
|
|
392
|
+
lines = open(f, encoding="utf-8").read().splitlines()
|
|
393
|
+
except Exception:
|
|
394
|
+
continue
|
|
395
|
+
for line in lines:
|
|
396
|
+
if not line.strip():
|
|
397
|
+
continue
|
|
398
|
+
try:
|
|
399
|
+
d = json.loads(line)
|
|
400
|
+
except Exception:
|
|
401
|
+
continue
|
|
402
|
+
t = d.get("type")
|
|
403
|
+
tms = toms(d.get("timestamp"))
|
|
404
|
+
if tms:
|
|
405
|
+
created = tms if not created else min(created, tms)
|
|
406
|
+
updated = max(updated, tms)
|
|
407
|
+
if t == "session":
|
|
408
|
+
cwd = d.get("cwd") or cwd
|
|
409
|
+
sid = d.get("id") or sid
|
|
410
|
+
continue
|
|
411
|
+
if t == "model_change":
|
|
412
|
+
cur_model = d.get("modelId") or cur_model
|
|
413
|
+
continue
|
|
414
|
+
if t != "message":
|
|
415
|
+
continue
|
|
416
|
+
m = d.get("message") or {}
|
|
417
|
+
role = m.get("role")
|
|
418
|
+
if role == "system":
|
|
419
|
+
continue
|
|
420
|
+
model = m.get("model") or (cur_model if role == "assistant" else "")
|
|
421
|
+
if model and role == "assistant":
|
|
422
|
+
models.add(model)
|
|
423
|
+
content = m.get("content")
|
|
424
|
+
text_parts, think, tools = [], [], []
|
|
425
|
+
if isinstance(content, str):
|
|
426
|
+
text_parts.append(content)
|
|
427
|
+
elif isinstance(content, list):
|
|
428
|
+
for blk in content:
|
|
429
|
+
if not isinstance(blk, dict):
|
|
430
|
+
continue
|
|
431
|
+
bt = blk.get("type")
|
|
432
|
+
if bt == "text":
|
|
433
|
+
text_parts.append(blk.get("text", ""))
|
|
434
|
+
elif bt in ("thinking", "reasoning"):
|
|
435
|
+
think.append(blk.get("text", "") or blk.get("thinking", ""))
|
|
436
|
+
elif bt in ("tool_use", "tool_call", "tool"):
|
|
437
|
+
tools.append({"name": blk.get("name") or blk.get("tool"),
|
|
438
|
+
"args": blk.get("input") or blk.get("args"), "result": None})
|
|
439
|
+
elif bt in ("tool_result",):
|
|
440
|
+
tools.append({"name": "result", "args": "",
|
|
441
|
+
"result": json.dumps(blk.get("content"))[:1000]})
|
|
442
|
+
msg = {"role": role, "model": model if role == "assistant" else "",
|
|
443
|
+
"thinking": "\n\n".join(x for x in think if x),
|
|
444
|
+
"text": "\n".join(tp for tp in text_parts if tp).strip(), "tools": tools}
|
|
445
|
+
if msg["text"] or msg["thinking"] or msg["tools"]:
|
|
446
|
+
messages.append(msg)
|
|
447
|
+
if not title and role == "user" and msg["text"]:
|
|
448
|
+
title = msg["text"].splitlines()[0][:80]
|
|
449
|
+
if not messages:
|
|
450
|
+
continue
|
|
451
|
+
yield {"harness": "pi", "id": sid, "project": canonical_project(cwd),
|
|
452
|
+
"source_path": cwd, "title": title, "created": created,
|
|
453
|
+
"updated": updated, "models": models, "messages": messages}
|
|
454
|
+
|
|
455
|
+
# -------------------------------------------------------------- OPENCODE -----
|
|
456
|
+
|
|
457
|
+
def opencode_chats():
|
|
458
|
+
db = opencode_db_path()
|
|
459
|
+
if not db or not os.path.isfile(db):
|
|
460
|
+
return
|
|
461
|
+
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
462
|
+
con.row_factory = sqlite3.Row
|
|
463
|
+
sessions = {}
|
|
464
|
+
for tbl in ("session", "session_v2"): # v2 overrides
|
|
465
|
+
try:
|
|
466
|
+
for r in con.execute(f"SELECT * FROM {tbl}"):
|
|
467
|
+
sessions[r["id"]] = dict(r)
|
|
468
|
+
except Exception:
|
|
469
|
+
pass
|
|
470
|
+
for sid, s in sessions.items():
|
|
471
|
+
directory = s.get("directory") or ""
|
|
472
|
+
title = s.get("title") or ""
|
|
473
|
+
created = toms(s.get("time_created"))
|
|
474
|
+
updated = toms(s.get("time_updated")) or created
|
|
475
|
+
sess_model = ""
|
|
476
|
+
mj = s.get("model")
|
|
477
|
+
if mj:
|
|
478
|
+
try:
|
|
479
|
+
sess_model = json.loads(mj).get("id", "")
|
|
480
|
+
except Exception:
|
|
481
|
+
sess_model = ""
|
|
482
|
+
messages = []
|
|
483
|
+
models = set()
|
|
484
|
+
if sess_model:
|
|
485
|
+
models.add(sess_model)
|
|
486
|
+
try:
|
|
487
|
+
rows = con.execute(
|
|
488
|
+
"SELECT type,data FROM session_message WHERE session_id=? ORDER BY seq", (sid,)
|
|
489
|
+
).fetchall()
|
|
490
|
+
except Exception:
|
|
491
|
+
rows = []
|
|
492
|
+
for r in rows:
|
|
493
|
+
typ = r["type"]
|
|
494
|
+
if typ not in ("user", "assistant"):
|
|
495
|
+
continue
|
|
496
|
+
try:
|
|
497
|
+
data = json.loads(r["data"])
|
|
498
|
+
except Exception:
|
|
499
|
+
continue
|
|
500
|
+
model = ""
|
|
501
|
+
mm = data.get("model")
|
|
502
|
+
if isinstance(mm, dict):
|
|
503
|
+
model = mm.get("id", "")
|
|
504
|
+
if model:
|
|
505
|
+
models.add(model)
|
|
506
|
+
text_parts, think, tools = [], [], []
|
|
507
|
+
content = data.get("content")
|
|
508
|
+
if isinstance(data.get("text"), str) and not content:
|
|
509
|
+
text_parts.append(data["text"])
|
|
510
|
+
if isinstance(content, list):
|
|
511
|
+
for blk in content:
|
|
512
|
+
if not isinstance(blk, dict):
|
|
513
|
+
continue
|
|
514
|
+
bt = blk.get("type")
|
|
515
|
+
if bt == "text":
|
|
516
|
+
text_parts.append(blk.get("text", ""))
|
|
517
|
+
elif bt == "reasoning":
|
|
518
|
+
think.append(blk.get("text", ""))
|
|
519
|
+
elif bt == "tool":
|
|
520
|
+
st = blk.get("state") or {}
|
|
521
|
+
tools.append({"name": blk.get("tool"),
|
|
522
|
+
"args": (st.get("input") if isinstance(st, dict) else None),
|
|
523
|
+
"result": None})
|
|
524
|
+
msg = {"role": typ, "model": model, "thinking": "\n\n".join(x for x in think if x),
|
|
525
|
+
"text": "\n".join(tp for tp in text_parts if tp).strip(), "tools": tools}
|
|
526
|
+
if msg["text"] or msg["thinking"] or msg["tools"]:
|
|
527
|
+
messages.append(msg)
|
|
528
|
+
if not title and typ == "user" and msg["text"]:
|
|
529
|
+
title = msg["text"].splitlines()[0][:80]
|
|
530
|
+
if not messages:
|
|
531
|
+
continue
|
|
532
|
+
yield {"harness": "opencode", "id": sid, "project": canonical_project(directory),
|
|
533
|
+
"source_path": directory, "title": title, "created": created,
|
|
534
|
+
"updated": updated, "models": models, "messages": messages}
|
|
535
|
+
|
|
536
|
+
# ----------------------------------------------------------------- CODEX -----
|
|
537
|
+
|
|
538
|
+
def _codex_text(parts):
|
|
539
|
+
out = []
|
|
540
|
+
for p in (parts or []):
|
|
541
|
+
if isinstance(p, dict):
|
|
542
|
+
t = p.get("text") or p.get("input_text") or p.get("output_text")
|
|
543
|
+
if isinstance(t, str) and t.strip():
|
|
544
|
+
out.append(t)
|
|
545
|
+
elif isinstance(p, str):
|
|
546
|
+
out.append(p)
|
|
547
|
+
return "\n".join(out)
|
|
548
|
+
|
|
549
|
+
def codex_chats():
|
|
550
|
+
roots = [os.path.join(HOME, ".codex/sessions"), os.path.join(HOME, ".codex/archived_sessions")]
|
|
551
|
+
files = []
|
|
552
|
+
for r in roots:
|
|
553
|
+
files += glob.glob(os.path.join(r, "**", "*.jsonl"), recursive=True)
|
|
554
|
+
for f in files:
|
|
555
|
+
cwd = ""; sid = os.path.splitext(os.path.basename(f))[0]
|
|
556
|
+
created = updated = 0; title = ""; model = ""
|
|
557
|
+
messages = []; models = set(); last_tool = None
|
|
558
|
+
try:
|
|
559
|
+
lines = open(f, encoding="utf-8").read().splitlines()
|
|
560
|
+
except Exception:
|
|
561
|
+
continue
|
|
562
|
+
for line in lines:
|
|
563
|
+
if not line.strip():
|
|
564
|
+
continue
|
|
565
|
+
try:
|
|
566
|
+
d = json.loads(line)
|
|
567
|
+
except Exception:
|
|
568
|
+
continue
|
|
569
|
+
tms = toms(d.get("timestamp"))
|
|
570
|
+
if tms:
|
|
571
|
+
created = tms if not created else min(created, tms)
|
|
572
|
+
updated = max(updated, tms)
|
|
573
|
+
typ = d.get("type")
|
|
574
|
+
pl = d.get("payload") or {}
|
|
575
|
+
if typ == "session_meta":
|
|
576
|
+
cwd = pl.get("cwd") or cwd
|
|
577
|
+
sid = pl.get("id") or sid
|
|
578
|
+
model = pl.get("model") or model
|
|
579
|
+
continue
|
|
580
|
+
if typ != "response_item" or not isinstance(pl, dict):
|
|
581
|
+
continue
|
|
582
|
+
pt = pl.get("type")
|
|
583
|
+
if pt == "message":
|
|
584
|
+
role = pl.get("role")
|
|
585
|
+
if role not in ("user", "assistant"):
|
|
586
|
+
continue
|
|
587
|
+
txt = _codex_text(pl.get("content"))
|
|
588
|
+
st = txt.strip()
|
|
589
|
+
# skip Codex's injected context turns (not real conversation)
|
|
590
|
+
if role == "user" and (st[:1] == "<" or st.startswith("# AGENTS.md")
|
|
591
|
+
or st.startswith("environment_context")
|
|
592
|
+
or st.startswith("## My request")):
|
|
593
|
+
# some wrap the real ask in <user_instructions>/<user_query>
|
|
594
|
+
m = re.search(r"<(?:user_query|user_instructions|user_message)>\s*(.*?)\s*</", txt, re.S)
|
|
595
|
+
if m and m.group(1).strip():
|
|
596
|
+
txt = m.group(1).strip(); st = txt
|
|
597
|
+
else:
|
|
598
|
+
continue
|
|
599
|
+
if st:
|
|
600
|
+
messages.append({"role": role, "model": model if role == "assistant" else "",
|
|
601
|
+
"thinking": "", "text": txt, "tools": []})
|
|
602
|
+
last_tool = None
|
|
603
|
+
if not title and role == "user":
|
|
604
|
+
title = st.splitlines()[0][:80]
|
|
605
|
+
elif pt == "reasoning":
|
|
606
|
+
th = _codex_text(pl.get("summary")) or _codex_text(pl.get("content"))
|
|
607
|
+
if th.strip():
|
|
608
|
+
messages.append({"role": "assistant", "model": model, "thinking": th,
|
|
609
|
+
"text": "", "tools": []})
|
|
610
|
+
last_tool = None
|
|
611
|
+
elif pt == "function_call":
|
|
612
|
+
tool = {"name": pl.get("name"), "args": pl.get("arguments"), "result": None}
|
|
613
|
+
messages.append({"role": "assistant", "model": model, "thinking": "",
|
|
614
|
+
"text": "", "tools": [tool]})
|
|
615
|
+
last_tool = tool
|
|
616
|
+
elif pt == "function_call_output" and last_tool is not None:
|
|
617
|
+
out = pl.get("output")
|
|
618
|
+
last_tool["result"] = out if isinstance(out, str) else json.dumps(out)[:1000]
|
|
619
|
+
if model:
|
|
620
|
+
models.add(model)
|
|
621
|
+
if not messages:
|
|
622
|
+
continue
|
|
623
|
+
yield {"harness": "codex", "id": sid, "project": canonical_project(cwd),
|
|
624
|
+
"source_path": cwd, "title": title, "created": created,
|
|
625
|
+
"updated": updated, "models": models, "messages": messages}
|
|
626
|
+
|
|
627
|
+
# ---------------------------------------------------------------- GEMINI -----
|
|
628
|
+
|
|
629
|
+
def _gemini_projects():
|
|
630
|
+
try:
|
|
631
|
+
d = json.load(open(os.path.join(HOME, ".gemini/projects.json")))
|
|
632
|
+
return {v: k for k, v in (d.get("projects") or {}).items()} # name -> path
|
|
633
|
+
except Exception:
|
|
634
|
+
return {}
|
|
635
|
+
|
|
636
|
+
def gemini_chats():
|
|
637
|
+
tmp = os.path.join(HOME, ".gemini/tmp")
|
|
638
|
+
name2path = _gemini_projects()
|
|
639
|
+
for d in glob.glob(os.path.join(tmp, "*")):
|
|
640
|
+
if not os.path.isdir(d):
|
|
641
|
+
continue
|
|
642
|
+
key = os.path.basename(d)
|
|
643
|
+
# resolve cwd: named project map, else parse from a checkpoint setup msg
|
|
644
|
+
cwd = name2path.get(key, "")
|
|
645
|
+
checkpoints = sorted(glob.glob(os.path.join(d, "checkpoint-*.json")))
|
|
646
|
+
emitted = False
|
|
647
|
+
for cp in checkpoints:
|
|
648
|
+
try:
|
|
649
|
+
data = json.load(open(cp, encoding="utf-8"))
|
|
650
|
+
except Exception:
|
|
651
|
+
continue
|
|
652
|
+
hist = data.get("history") if isinstance(data, dict) else data
|
|
653
|
+
if not isinstance(hist, list):
|
|
654
|
+
continue
|
|
655
|
+
messages = []; title = ""
|
|
656
|
+
for h in hist:
|
|
657
|
+
if not isinstance(h, dict):
|
|
658
|
+
continue
|
|
659
|
+
role = "assistant" if h.get("role") == "model" else h.get("role")
|
|
660
|
+
if role not in ("user", "assistant"):
|
|
661
|
+
continue
|
|
662
|
+
text_parts, think, tools = [], [], []
|
|
663
|
+
for p in (h.get("parts") or []):
|
|
664
|
+
if not isinstance(p, dict):
|
|
665
|
+
continue
|
|
666
|
+
if p.get("thought"):
|
|
667
|
+
think.append(p.get("text", ""))
|
|
668
|
+
elif "text" in p:
|
|
669
|
+
text_parts.append(p.get("text", ""))
|
|
670
|
+
elif "functionCall" in p:
|
|
671
|
+
fc = p["functionCall"]
|
|
672
|
+
tools.append({"name": fc.get("name"), "args": fc.get("args"), "result": None})
|
|
673
|
+
elif "functionResponse" in p:
|
|
674
|
+
fr = p["functionResponse"]
|
|
675
|
+
if tools:
|
|
676
|
+
tools[-1]["result"] = json.dumps(fr.get("response"))[:1000]
|
|
677
|
+
txt = "\n".join(t for t in text_parts if t).strip()
|
|
678
|
+
# skip the Gemini CLI context-setup preamble
|
|
679
|
+
if role == "user" and txt.startswith("This is the Gemini CLI"):
|
|
680
|
+
m = re.search(r"working in the project directory:\s*(\S+)", txt)
|
|
681
|
+
if m and not cwd:
|
|
682
|
+
cwd = m.group(1)
|
|
683
|
+
continue
|
|
684
|
+
if not (txt or think or tools):
|
|
685
|
+
continue
|
|
686
|
+
messages.append({"role": role, "model": "", "thinking": "\n\n".join(think),
|
|
687
|
+
"text": txt, "tools": tools})
|
|
688
|
+
if not title and role == "user" and txt:
|
|
689
|
+
title = txt.splitlines()[0][:80]
|
|
690
|
+
if not messages:
|
|
691
|
+
continue
|
|
692
|
+
mt = int(os.path.getmtime(cp) * 1000)
|
|
693
|
+
tag = os.path.basename(cp)[len("checkpoint-"):-len(".json")] or "chat"
|
|
694
|
+
yield {"harness": "gemini", "id": f"{key[:12]}-{tag}", "project": canonical_project(cwd),
|
|
695
|
+
"source_path": cwd, "title": title, "created": mt, "updated": mt,
|
|
696
|
+
"models": set(), "messages": messages}
|
|
697
|
+
emitted = True
|
|
698
|
+
if emitted:
|
|
699
|
+
continue
|
|
700
|
+
# fallback: logs.json (user prompts only) grouped by sessionId
|
|
701
|
+
lg = os.path.join(d, "logs.json")
|
|
702
|
+
if not os.path.isfile(lg):
|
|
703
|
+
continue
|
|
704
|
+
try:
|
|
705
|
+
entries = json.load(open(lg, encoding="utf-8"))
|
|
706
|
+
except Exception:
|
|
707
|
+
continue
|
|
708
|
+
if not isinstance(entries, list) or not entries:
|
|
709
|
+
continue
|
|
710
|
+
bysess = defaultdict(list)
|
|
711
|
+
for e in entries:
|
|
712
|
+
if isinstance(e, dict) and e.get("message"):
|
|
713
|
+
bysess[e.get("sessionId", key)].append(e)
|
|
714
|
+
for sess, es in bysess.items():
|
|
715
|
+
es.sort(key=lambda x: x.get("messageId", 0))
|
|
716
|
+
messages = [{"role": "user", "model": "", "thinking": "",
|
|
717
|
+
"text": str(e.get("message", "")), "tools": []} for e in es]
|
|
718
|
+
if not messages:
|
|
719
|
+
continue
|
|
720
|
+
t0 = toms(es[0].get("timestamp")) or int(os.path.getmtime(lg) * 1000)
|
|
721
|
+
t1 = toms(es[-1].get("timestamp")) or t0
|
|
722
|
+
yield {"harness": "gemini", "id": str(sess)[:12], "project": canonical_project(cwd),
|
|
723
|
+
"source_path": cwd, "title": messages[0]["text"].splitlines()[0][:80],
|
|
724
|
+
"created": t0, "updated": t1, "models": set(), "messages": messages}
|
|
725
|
+
|
|
726
|
+
# ----------------------------------------------------------- CURSOR AGENT ----
|
|
727
|
+
|
|
728
|
+
def cursor_agent_chats():
|
|
729
|
+
"""Cursor Agent CLI (cursor-agent): ~/.cursor/acp-sessions/<uuid>/ with a
|
|
730
|
+
git-like blob store (store.db) + meta.json. Message blobs are JSON
|
|
731
|
+
{role, content:[parts]}; ordered by insertion (rowid)."""
|
|
732
|
+
root = os.path.join(HOME, ".cursor/acp-sessions")
|
|
733
|
+
for d in glob.glob(os.path.join(root, "*")):
|
|
734
|
+
db = os.path.join(d, "store.db")
|
|
735
|
+
if not os.path.isfile(db):
|
|
736
|
+
continue
|
|
737
|
+
sid = os.path.basename(d)
|
|
738
|
+
cwd = ""; title = ""
|
|
739
|
+
try:
|
|
740
|
+
mj = json.load(open(os.path.join(d, "meta.json")))
|
|
741
|
+
cwd = mj.get("cwd", ""); title = mj.get("title", "")
|
|
742
|
+
except Exception:
|
|
743
|
+
pass
|
|
744
|
+
created = 0
|
|
745
|
+
try:
|
|
746
|
+
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
747
|
+
except Exception:
|
|
748
|
+
continue
|
|
749
|
+
try:
|
|
750
|
+
r = con.execute("SELECT value FROM meta WHERE key='0'").fetchone()
|
|
751
|
+
if r:
|
|
752
|
+
try:
|
|
753
|
+
mm = json.loads(bytes.fromhex(r[0]).decode("utf-8", "replace"))
|
|
754
|
+
title = title or mm.get("name", "")
|
|
755
|
+
created = toms(mm.get("createdAt"))
|
|
756
|
+
except Exception:
|
|
757
|
+
pass
|
|
758
|
+
except Exception:
|
|
759
|
+
pass
|
|
760
|
+
updated = int(os.path.getmtime(db) * 1000)
|
|
761
|
+
created = created or updated
|
|
762
|
+
messages = []
|
|
763
|
+
try:
|
|
764
|
+
rows = con.execute("SELECT data FROM blobs ORDER BY rowid").fetchall()
|
|
765
|
+
except Exception:
|
|
766
|
+
rows = []
|
|
767
|
+
con.close()
|
|
768
|
+
for (data,) in rows:
|
|
769
|
+
if not data:
|
|
770
|
+
continue
|
|
771
|
+
try:
|
|
772
|
+
j = json.loads(bytes(data).decode("utf-8", "replace"))
|
|
773
|
+
except Exception:
|
|
774
|
+
continue
|
|
775
|
+
if not isinstance(j, dict) or j.get("role") not in ("user", "assistant"):
|
|
776
|
+
continue
|
|
777
|
+
role = j["role"]
|
|
778
|
+
text_parts, think, tools = [], [], []
|
|
779
|
+
for p in (j.get("content") or []):
|
|
780
|
+
if not isinstance(p, dict):
|
|
781
|
+
continue
|
|
782
|
+
pt = p.get("type")
|
|
783
|
+
if pt == "text":
|
|
784
|
+
t = p.get("text", "")
|
|
785
|
+
t = re.sub(r"</?user_(?:query|response|message)>", "", t).strip()
|
|
786
|
+
if t:
|
|
787
|
+
text_parts.append(t)
|
|
788
|
+
elif pt == "reasoning":
|
|
789
|
+
if p.get("text"):
|
|
790
|
+
think.append(p["text"])
|
|
791
|
+
elif pt == "tool-call":
|
|
792
|
+
tools.append({"name": p.get("toolName"), "args": p.get("args"), "result": None})
|
|
793
|
+
msg = {"role": role, "model": "", "thinking": "\n\n".join(think),
|
|
794
|
+
"text": "\n".join(text_parts).strip(), "tools": tools}
|
|
795
|
+
if msg["text"] or msg["thinking"] or msg["tools"]:
|
|
796
|
+
messages.append(msg)
|
|
797
|
+
if not title and role == "user" and msg["text"]:
|
|
798
|
+
title = msg["text"].splitlines()[0][:80]
|
|
799
|
+
if not messages:
|
|
800
|
+
continue
|
|
801
|
+
yield {"harness": "cursor-agent", "id": sid, "project": canonical_project(cwd),
|
|
802
|
+
"source_path": cwd, "title": title, "created": created,
|
|
803
|
+
"updated": updated, "models": set(), "messages": messages}
|
|
804
|
+
|
|
805
|
+
# ------------------------------------------------------------- ANTIGRAVITY ---
|
|
806
|
+
|
|
807
|
+
def antigravity_chats():
|
|
808
|
+
"""Antigravity (Google's agentic IDE) stores its 'Cascade' agent conversations
|
|
809
|
+
in an opaque protobuf LevelDB (Application Support/Antigravity/shared_proto_db)
|
|
810
|
+
with an empty JSON chat index — not locally parseable yet (and likely cloud
|
|
811
|
+
synced). Stub kept so the harness is wired for a future decoder."""
|
|
812
|
+
return
|
|
813
|
+
yield # pragma: no cover
|
|
814
|
+
|
|
815
|
+
# --------------------------------------------------------------- orchestrate -
|
|
816
|
+
|
|
817
|
+
ADAPTERS = [cursor_chats, claude_code_chats, opencode_chats, pi_chats,
|
|
818
|
+
codex_chats, gemini_chats, cursor_agent_chats, antigravity_chats]
|
|
819
|
+
|
|
820
|
+
FM_RE = re.compile(r"^(\w+):\s*(.*)$")
|
|
821
|
+
|
|
822
|
+
def toms_iso(v):
|
|
823
|
+
if not v:
|
|
824
|
+
return 0
|
|
825
|
+
return toms(v)
|
|
826
|
+
|
|
827
|
+
def read_frontmatter(path):
|
|
828
|
+
"""Parse the YAML-ish frontmatter block of a chat file (cheap, top of file)."""
|
|
829
|
+
fm = {}
|
|
830
|
+
try:
|
|
831
|
+
with open(path) as f:
|
|
832
|
+
if f.readline().strip() != "---":
|
|
833
|
+
return fm
|
|
834
|
+
for _ in range(40):
|
|
835
|
+
line = f.readline()
|
|
836
|
+
if not line or line.strip() == "---":
|
|
837
|
+
break
|
|
838
|
+
m = FM_RE.match(line.rstrip("\n"))
|
|
839
|
+
if m:
|
|
840
|
+
k, v = m.group(1), m.group(2).strip()
|
|
841
|
+
if v[:1] in "\"'" or v[:1] == "[":
|
|
842
|
+
try:
|
|
843
|
+
v = json.loads(v)
|
|
844
|
+
except Exception:
|
|
845
|
+
v = v.strip("\"'")
|
|
846
|
+
fm[k] = v
|
|
847
|
+
except Exception:
|
|
848
|
+
pass
|
|
849
|
+
return fm
|
|
850
|
+
|
|
851
|
+
def main():
|
|
852
|
+
os.makedirs(PROJ_DIR, exist_ok=True)
|
|
853
|
+
stats = defaultdict(int)
|
|
854
|
+
per_harness = defaultdict(int)
|
|
855
|
+
keep = set() # abs paths this client produced this run
|
|
856
|
+
|
|
857
|
+
# 1) export THIS client's chats
|
|
858
|
+
for adapter in ADAPTERS:
|
|
859
|
+
try:
|
|
860
|
+
for c in adapter():
|
|
861
|
+
c.setdefault("client", CLIENT)
|
|
862
|
+
c.setdefault("platform", PLATFORM)
|
|
863
|
+
md = render_chat(c)
|
|
864
|
+
if not md:
|
|
865
|
+
stats["empty"] += 1
|
|
866
|
+
continue
|
|
867
|
+
proj = c["project"] or "_unknown"
|
|
868
|
+
fname = f"{datestr(c['created'])}_{CLIENT}_{slug(c['title'] or c['id'])}_{c['id'][:8]}.md"
|
|
869
|
+
fpath = os.path.join(PROJ_DIR, proj, c["harness"], fname)
|
|
870
|
+
keep.add(os.path.abspath(fpath))
|
|
871
|
+
stats[write_if_changed(fpath, md)] += 1
|
|
872
|
+
per_harness[c["harness"]] += 1
|
|
873
|
+
except Exception as e:
|
|
874
|
+
print(f"adapter {adapter.__name__} error: {e}")
|
|
875
|
+
|
|
876
|
+
# 2) prune ONLY this client's stale files (never touch other machines' chats)
|
|
877
|
+
for p in glob.glob(os.path.join(PROJ_DIR, "*", "*", "*.md")):
|
|
878
|
+
if os.path.abspath(p) in keep:
|
|
879
|
+
continue
|
|
880
|
+
if read_frontmatter(p).get("client") == CLIENT:
|
|
881
|
+
try:
|
|
882
|
+
os.remove(p)
|
|
883
|
+
except Exception:
|
|
884
|
+
pass
|
|
885
|
+
|
|
886
|
+
# 3) rebuild indexes from the WHOLE repo (union of all clients, incl. git-pulled)
|
|
887
|
+
index = defaultdict(list)
|
|
888
|
+
clients_seen = set()
|
|
889
|
+
harness_totals = defaultdict(int)
|
|
890
|
+
for p in glob.glob(os.path.join(PROJ_DIR, "*", "*", "*.md")):
|
|
891
|
+
fm = read_frontmatter(p)
|
|
892
|
+
proj = os.path.basename(os.path.dirname(os.path.dirname(p)))
|
|
893
|
+
harness = fm.get("harness") or os.path.basename(os.path.dirname(p))
|
|
894
|
+
cl = fm.get("client") or "?"
|
|
895
|
+
clients_seen.add(cl)
|
|
896
|
+
harness_totals[harness] += 1
|
|
897
|
+
index[proj].append({
|
|
898
|
+
"updated": toms_iso(fm.get("updated")) or toms_iso(fm.get("created")),
|
|
899
|
+
"client": cl, "platform": fm.get("platform", ""), "harness": harness,
|
|
900
|
+
"title": str(fm.get("title") or ""), "msgs": fm.get("messages", ""),
|
|
901
|
+
"models": fm.get("models", ""),
|
|
902
|
+
"rel": os.path.join(harness, os.path.basename(p)),
|
|
903
|
+
})
|
|
904
|
+
|
|
905
|
+
for proj, entries in index.items():
|
|
906
|
+
entries.sort(key=lambda e: e["updated"], reverse=True)
|
|
907
|
+
lines = [f"# {proj} — chat memory", "",
|
|
908
|
+
f"{len(entries)} chats across harnesses & machines. Newest first.", "",
|
|
909
|
+
"| Last active | Client | Harness | Title | Msgs | Models | File |",
|
|
910
|
+
"|---|---|---|---|---|---|---|"]
|
|
911
|
+
for e in entries:
|
|
912
|
+
tt = e["title"].replace("|", "\\|")[:70]
|
|
913
|
+
lines.append(f"| {datestr(e['updated'])} | {e['client']} | {e['harness']} | {tt} | "
|
|
914
|
+
f"{e['msgs']} | {e['models']} | [open]({e['rel'].replace(' ', '%20')}) |")
|
|
915
|
+
write_if_changed(os.path.join(PROJ_DIR, proj, "INDEX.md"), "\n".join(lines) + "\n")
|
|
916
|
+
|
|
917
|
+
rows = sorted(((p, len(e), max((x["updated"] for x in e), default=0),
|
|
918
|
+
sorted({x["client"] for x in e})) for p, e in index.items()),
|
|
919
|
+
key=lambda r: r[2], reverse=True)
|
|
920
|
+
total = sum(len(e) for e in index.values())
|
|
921
|
+
g = ["# Chat memory — all projects", "",
|
|
922
|
+
f"Last sync: {datetime.datetime.now():%Y-%m-%d %H:%M} (ran on: {CLIENT}/{PLATFORM})",
|
|
923
|
+
f"Clients: {', '.join(sorted(clients_seen)) or '-'}",
|
|
924
|
+
f"Total chats: {total} (" + ", ".join(f"{h}={n}" for h, n in sorted(harness_totals.items())) + ")",
|
|
925
|
+
"",
|
|
926
|
+
"| Project | Chats | Last active | Clients | Index |", "|---|---|---|---|---|"]
|
|
927
|
+
for p, n, lu, cls in rows:
|
|
928
|
+
g.append(f"| {p} | {n} | {datestr(lu)} | {', '.join(cls)} | "
|
|
929
|
+
f"[projects/{p}/INDEX.md](projects/{p}/INDEX.md) |")
|
|
930
|
+
write_if_changed(os.path.join(OUT, "INDEX.md"), "\n".join(g) + "\n")
|
|
931
|
+
|
|
932
|
+
print(f"client={CLIENT} chats: new={stats['new']} updated={stats['updated']} "
|
|
933
|
+
f"unchanged={stats['unchanged']} empty={stats['empty']} | "
|
|
934
|
+
f"this-run by harness: {dict(per_harness)} | repo total: {total}")
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
if __name__ == "__main__":
|
|
938
|
+
main()
|