thread-archive 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thread_archive/__init__.py +34 -0
- thread_archive/_api.py +2235 -0
- thread_archive/_config.py +126 -0
- thread_archive/_importers/__init__.py +81 -0
- thread_archive/_importers/_continuation.py +136 -0
- thread_archive/_importers/_cursor.py +103 -0
- thread_archive/_importers/_events.py +244 -0
- thread_archive/_importers/_line_stream.py +193 -0
- thread_archive/_importers/_read.py +82 -0
- thread_archive/_importers/_result.py +17 -0
- thread_archive/_importers/_sidecar.py +113 -0
- thread_archive/_importers/_state.py +240 -0
- thread_archive/_importers/_titles.py +179 -0
- thread_archive/_importers/antigravity.py +254 -0
- thread_archive/_importers/claude_code.py +293 -0
- thread_archive/_importers/claude_science.py +330 -0
- thread_archive/_importers/cloth.py +20 -0
- thread_archive/_importers/codex.py +324 -0
- thread_archive/_importers/cowork.py +107 -0
- thread_archive/_importers/cursor.py +432 -0
- thread_archive/_importers/exports.py +389 -0
- thread_archive/_importers/grok.py +600 -0
- thread_archive/_importers/opencode.py +538 -0
- thread_archive/_knowledge/__init__.py +67 -0
- thread_archive/_knowledge/_claims.py +116 -0
- thread_archive/_knowledge/_community.py +75 -0
- thread_archive/_knowledge/graph.py +208 -0
- thread_archive/_knowledge/materialize.py +212 -0
- thread_archive/_knowledge/write.py +438 -0
- thread_archive/_launchd.py +167 -0
- thread_archive/_mcp/__init__.py +1 -0
- thread_archive/_mcp/librarian.py +151 -0
- thread_archive/_mcp/server.py +307 -0
- thread_archive/_retrieval/__init__.py +280 -0
- thread_archive/_retrieval/_classify.py +80 -0
- thread_archive/_retrieval/_codex.py +157 -0
- thread_archive/_retrieval/_context.py +123 -0
- thread_archive/_retrieval/_extract.py +184 -0
- thread_archive/_retrieval/embed.py +186 -0
- thread_archive/_retrieval/format.py +149 -0
- thread_archive/_retrieval/fts.py +538 -0
- thread_archive/_retrieval/rank.py +228 -0
- thread_archive/_retrieval/read.py +908 -0
- thread_archive/_retrieval/rerank.py +143 -0
- thread_archive/_retrieval/vectors.py +611 -0
- thread_archive/_scripts/__init__.py +1 -0
- thread_archive/_scripts/backfill_recompute.py +328 -0
- thread_archive/_scripts/backfill_reconcile.py +296 -0
- thread_archive/_scripts/denamespace_dedup_keys.py +120 -0
- thread_archive/_scripts/recover_dropped_events.py +190 -0
- thread_archive/_scripts/repair_grok_tool_names.py +118 -0
- thread_archive/_scripts/repair_grok_tool_names_backup_20260704T155625Z.json +275 -0
- thread_archive/_scripts/repair_grok_tool_names_plan_20260704.json +435 -0
- thread_archive/_setup/__init__.py +12 -0
- thread_archive/_setup/clients.py +89 -0
- thread_archive/_setup/wizard.py +474 -0
- thread_archive/_store/__init__.py +45 -0
- thread_archive/_store/_base.py +173 -0
- thread_archive/_store/_defaults.py +57 -0
- thread_archive/_store/_types.py +38 -0
- thread_archive/_store/models.py +370 -0
- thread_archive/_store/schema.py +84 -0
- thread_archive/_thread_import/__init__.py +40 -0
- thread_archive/_thread_import/api.py +78 -0
- thread_archive/_thread_import/event_builder.py +810 -0
- thread_archive/_thread_import/exporters/__init__.py +9 -0
- thread_archive/_thread_import/exporters/_cursor_kv_mixin.py +166 -0
- thread_archive/_thread_import/exporters/_cursor_parse_mixin.py +149 -0
- thread_archive/_thread_import/exporters/cursor.py +582 -0
- thread_archive/_thread_import/exporters/cursor_parse.py +145 -0
- thread_archive/_thread_import/parsers/__init__.py +191 -0
- thread_archive/_thread_import/parsers/base.py +698 -0
- thread_archive/_thread_import/parsers/chatgpt.py +722 -0
- thread_archive/_thread_import/parsers/chatgpt_content.py +332 -0
- thread_archive/_thread_import/parsers/claude.py +443 -0
- thread_archive/_thread_import/parsers/claude_code.py +1035 -0
- thread_archive/_thread_import/parsers/claude_code_blocks.py +415 -0
- thread_archive/_thread_import/parsers/claude_code_ide.py +122 -0
- thread_archive/_thread_import/parsers/claude_code_sessions.py +90 -0
- thread_archive/_thread_import/parsers/config/__init__.py +26 -0
- thread_archive/_thread_import/parsers/config/base.py +220 -0
- thread_archive/_thread_import/parsers/cursor.py +538 -0
- thread_archive/_thread_import/parsers/cursor_blocks.py +365 -0
- thread_archive/_thread_import/parsers/pipeline/__init__.py +29 -0
- thread_archive/_thread_import/parsers/pipeline/base.py +251 -0
- thread_archive/_thread_import/parsers/pipeline/interfaces.py +191 -0
- thread_archive/_thread_import/parsers/transformers/__init__.py +22 -0
- thread_archive/_thread_import/parsers/transformers/active_path.py +87 -0
- thread_archive/_thread_import/parsers/transformers/coalescing.py +389 -0
- thread_archive/_thread_import/parsers/transformers/ide_context.py +158 -0
- thread_archive/_thread_import/parsers/transformers/thinking_merge.py +68 -0
- thread_archive/_thread_import/parsers/types/__init__.py +56 -0
- thread_archive/_thread_import/parsers/types/chatgpt.py +99 -0
- thread_archive/_thread_import/parsers/types/claude.py +120 -0
- thread_archive/_thread_import/parsers/types/claude_code.py +139 -0
- thread_archive/_thread_import/parsers/types/cursor.py +109 -0
- thread_archive/_thread_import/parsers/validators/__init__.py +83 -0
- thread_archive/_thread_import/parsers/validators/base.py +168 -0
- thread_archive/_thread_import/parsers/validators/content.py +80 -0
- thread_archive/_thread_import/parsers/validators/referential.py +57 -0
- thread_archive/_thread_import/parsers/validators/thinking.py +143 -0
- thread_archive/_thread_import/parsers/validators/types.py +87 -0
- thread_archive/_thread_import/schemas/__init__.py +231 -0
- thread_archive/_thread_import/schemas/chatgpt/v1.json +197 -0
- thread_archive/_thread_import/schemas/chatgpt/v2.json +203 -0
- thread_archive/_thread_import/schemas/claude/v1.json +188 -0
- thread_archive/_thread_import/schemas/claude_code/v1.json +183 -0
- thread_archive/_thread_import/schemas/cursor/v1.json +143 -0
- thread_archive/_thread_import/schemas/cursor/v2.json +200 -0
- thread_archive/_thread_import/timestamps.py +57 -0
- thread_archive/_thread_import/tool_names.py +48 -0
- thread_archive/_truth/__init__.py +40 -0
- thread_archive/_truth/jsonl_log.py +2340 -0
- thread_archive/_truth/repair.py +322 -0
- thread_archive/_watcher/__init__.py +57 -0
- thread_archive/_watcher/base.py +65 -0
- thread_archive/_watcher/daemon.py +289 -0
- thread_archive/_watcher/export_drop.py +203 -0
- thread_archive/_watcher/exthost.py +316 -0
- thread_archive/_watcher/lazy.py +117 -0
- thread_archive/_watcher/sources.py +616 -0
- thread_archive/_web/__init__.py +15 -0
- thread_archive/_web/server.py +299 -0
- thread_archive/_web/static/assets/index-DECSH1Mz.css +10 -0
- thread_archive/_web/static/assets/index-Djq0FK0y.js +101 -0
- thread_archive/_web/static/index.html +13 -0
- thread_archive/cli.py +746 -0
- thread_archive-0.0.2.dist-info/METADATA +296 -0
- thread_archive-0.0.2.dist-info/RECORD +132 -0
- thread_archive-0.0.2.dist-info/WHEEL +4 -0
- thread_archive-0.0.2.dist-info/entry_points.txt +6 -0
- thread_archive-0.0.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""Claude Code incremental import.
|
|
2
|
+
|
|
3
|
+
Reduced to the conversation-archive essentials and run as a single atomic
|
|
4
|
+
transaction. The watermark logic lives in :mod:`._cursor`: a content digest proves the
|
|
5
|
+
file was only appended to, and the line cursor (``new_lines = all_lines[start_line:]``)
|
|
6
|
+
then feeds just the grown tail into the importer — a source rewritten under the cursor
|
|
7
|
+
rewinds and re-imports whole instead. Idempotence across re-reads (and across a
|
|
8
|
+
rewind) is guaranteed by the dedup_key membership check in :func:`import_lines`.
|
|
9
|
+
|
|
10
|
+
**Atomicity:** the whole import — thread create, events, watermark — commits in one
|
|
11
|
+
session, so "the watermark advances only after a successful import" holds by
|
|
12
|
+
construction.
|
|
13
|
+
|
|
14
|
+
Deferred (enhancements, not core): compaction continuation/fork detection, subagent
|
|
15
|
+
session handling, sidecar hook context, batch-thread adoption, description/models
|
|
16
|
+
backfill, and rich title extraction.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import logging
|
|
22
|
+
from dataclasses import replace
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Optional
|
|
25
|
+
|
|
26
|
+
from thread_archive._thread_import import DefaultEventBuilder
|
|
27
|
+
from thread_archive._thread_import.parsers.claude_code import ClaudeCodeParser
|
|
28
|
+
|
|
29
|
+
from .._store import Thread, get_session
|
|
30
|
+
from ._continuation import resolve_continuation_thread
|
|
31
|
+
from ._cursor import resolve_source_cursor
|
|
32
|
+
from ._events import import_lines
|
|
33
|
+
from ._read import parse_session_lines, read_source_bytes
|
|
34
|
+
from ._result import IncrementalImportResult
|
|
35
|
+
from ._sidecar import import_sidecar_lines, read_sidecar_lines
|
|
36
|
+
from ._state import (
|
|
37
|
+
adopt_if_unwatermarked,
|
|
38
|
+
create_thread,
|
|
39
|
+
discard_new_thread,
|
|
40
|
+
get_import_state,
|
|
41
|
+
get_thread_by_source,
|
|
42
|
+
lookup_parent_thread,
|
|
43
|
+
set_thread_models_from_events,
|
|
44
|
+
update_thread_description,
|
|
45
|
+
update_thread_title,
|
|
46
|
+
upsert_import_state,
|
|
47
|
+
)
|
|
48
|
+
from ._titles import extract_description, extract_session_title, extract_title
|
|
49
|
+
|
|
50
|
+
logger = logging.getLogger(__name__)
|
|
51
|
+
|
|
52
|
+
SOURCE = "claude-code"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _is_subagent_source_id(source_id: str) -> bool:
|
|
56
|
+
"""True when this CC source_id names a Task-tool subagent transcript.
|
|
57
|
+
|
|
58
|
+
The watcher hands subagent files (``<project>/<uuid>/subagents/agent-*.jsonl``)
|
|
59
|
+
the same ``{project}:{stem}`` source_id as any other session; the stem keeps
|
|
60
|
+
Claude Code's ``agent-`` prefix, which is the marker that survives into the
|
|
61
|
+
import path. Subagent runs file as hidden ``thread_type='system'`` threads —
|
|
62
|
+
captured and searchable, but kept out of the archive sidebar."""
|
|
63
|
+
return source_id.rsplit(":", 1)[-1].startswith("agent-")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _cc_origin_metadata(all_lines: list[dict], source_id: str, *, session=None) -> Optional[dict]:
|
|
67
|
+
"""Origin-project metadata for a CC thread: the real ``cwd`` (first line that
|
|
68
|
+
carries one) + the munged ``project_dir`` from the source_id.
|
|
69
|
+
|
|
70
|
+
For a subagent transcript also stamp ``is_subagent`` plus the spawning session
|
|
71
|
+
(``parent_session_id`` from the lines' ``sessionId``) and the subagent's own
|
|
72
|
+
``agent_id`` (from ``agentId``), so the run can be joined back to its parent
|
|
73
|
+
thread. When a ``session`` is supplied and the parent resolves to no thread, the
|
|
74
|
+
gap is logged rather than lost silently."""
|
|
75
|
+
meta: dict[str, Any] = {}
|
|
76
|
+
for line in all_lines:
|
|
77
|
+
cwd = line.get("cwd")
|
|
78
|
+
if isinstance(cwd, str) and cwd:
|
|
79
|
+
meta["cwd"] = cwd
|
|
80
|
+
break
|
|
81
|
+
if ":" in source_id:
|
|
82
|
+
meta["project_dir"] = source_id.rsplit(":", 1)[0]
|
|
83
|
+
if _is_subagent_source_id(source_id):
|
|
84
|
+
meta["is_subagent"] = True
|
|
85
|
+
for line in all_lines:
|
|
86
|
+
if "agent_id" not in meta and line.get("agentId"):
|
|
87
|
+
meta["agent_id"] = line["agentId"]
|
|
88
|
+
if "parent_session_id" not in meta and line.get("sessionId"):
|
|
89
|
+
meta["parent_session_id"] = line["sessionId"]
|
|
90
|
+
if "agent_id" in meta and "parent_session_id" in meta:
|
|
91
|
+
break
|
|
92
|
+
parent_sid = meta.get("parent_session_id")
|
|
93
|
+
project_dir = meta.get("project_dir")
|
|
94
|
+
if session is not None and parent_sid and project_dir:
|
|
95
|
+
parent_source_id = f"{project_dir}:{parent_sid}"
|
|
96
|
+
if lookup_parent_thread(session, parent_source_id) is None:
|
|
97
|
+
logger.warning(
|
|
98
|
+
"Subagent %s: parent_session_id %s resolves to no thread (%s) — "
|
|
99
|
+
"run can't be joined back to its parent",
|
|
100
|
+
source_id, parent_sid, parent_source_id,
|
|
101
|
+
)
|
|
102
|
+
return meta or None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _import_cc(
|
|
106
|
+
session,
|
|
107
|
+
source_id: str,
|
|
108
|
+
all_lines: list[dict],
|
|
109
|
+
source_bytes: bytes,
|
|
110
|
+
parser: ClaudeCodeParser,
|
|
111
|
+
builder: DefaultEventBuilder,
|
|
112
|
+
*,
|
|
113
|
+
source: str = SOURCE,
|
|
114
|
+
title_override: Optional[str] = None,
|
|
115
|
+
) -> IncrementalImportResult:
|
|
116
|
+
import_state = get_import_state(session, source, source_id)
|
|
117
|
+
cursor = resolve_source_cursor(import_state, source_bytes, source=source, source_id=source_id)
|
|
118
|
+
current_file_size = len(source_bytes)
|
|
119
|
+
|
|
120
|
+
if cursor.unchanged and import_state:
|
|
121
|
+
# Byte-identical to the last import. Stamp the digest if this watermark predates
|
|
122
|
+
# it, so the next poll's append proof has something to check against.
|
|
123
|
+
import_state.last_content_hash = cursor.content_hash
|
|
124
|
+
return IncrementalImportResult(
|
|
125
|
+
lines_processed=0,
|
|
126
|
+
events_created=0,
|
|
127
|
+
thread_id=import_state.thread_id or 0,
|
|
128
|
+
is_new_thread=False,
|
|
129
|
+
last_message_uuid=import_state.last_message_uuid,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
total_lines = len(all_lines)
|
|
133
|
+
start_line = cursor.start_line
|
|
134
|
+
if start_line >= total_lines:
|
|
135
|
+
# The file changed but yielded no new parsed lines (a torn tail line the writer
|
|
136
|
+
# hasn't finished; a rewrite down to nothing). Advance size + digest so the
|
|
137
|
+
# completed line still reads as an append next poll — the line cursor tracks
|
|
138
|
+
# what actually parsed, so the line itself imports then.
|
|
139
|
+
if import_state:
|
|
140
|
+
import_state.last_line_count = total_lines
|
|
141
|
+
import_state.last_file_size = current_file_size
|
|
142
|
+
import_state.last_content_hash = cursor.content_hash
|
|
143
|
+
return IncrementalImportResult(
|
|
144
|
+
lines_processed=0,
|
|
145
|
+
events_created=0,
|
|
146
|
+
thread_id=(import_state.thread_id or 0) if import_state else 0,
|
|
147
|
+
is_new_thread=False,
|
|
148
|
+
last_message_uuid=import_state.last_message_uuid if import_state else None,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
new_lines = all_lines[start_line:]
|
|
152
|
+
|
|
153
|
+
# Resolve the thread: the watermark's thread_id wins, else lookup by source.
|
|
154
|
+
thread_id: Optional[int] = (
|
|
155
|
+
import_state.thread_id if (import_state and import_state.thread_id) else None
|
|
156
|
+
)
|
|
157
|
+
if thread_id is None:
|
|
158
|
+
existing = get_thread_by_source(session, source, source_id)
|
|
159
|
+
thread_id = existing.id if existing else None
|
|
160
|
+
|
|
161
|
+
if import_state is None and adopt_if_unwatermarked(
|
|
162
|
+
session, source=source, source_id=source_id,
|
|
163
|
+
thread_id=thread_id, total_lines=total_lines, file_size=current_file_size,
|
|
164
|
+
content_hash=cursor.content_hash,
|
|
165
|
+
):
|
|
166
|
+
return IncrementalImportResult(0, 0, thread_id or 0, False, None)
|
|
167
|
+
|
|
168
|
+
is_subagent = _is_subagent_source_id(source_id)
|
|
169
|
+
|
|
170
|
+
# Continuation / fork merge: a fresh CC session that continues an existing thread
|
|
171
|
+
# (created on context compaction) merges into it instead of fragmenting into its
|
|
172
|
+
# own thread. Subagents are independent transcripts; cloth-shaped sources (source
|
|
173
|
+
# != "claude-code") don't carry CC's compaction markers, so both skip it.
|
|
174
|
+
merged_continuation = False
|
|
175
|
+
if thread_id is None and source == SOURCE and not is_subagent:
|
|
176
|
+
parent_id = resolve_continuation_thread(session, all_lines, source_id, parser, builder)
|
|
177
|
+
if parent_id is not None:
|
|
178
|
+
thread_id = parent_id
|
|
179
|
+
merged_continuation = True
|
|
180
|
+
|
|
181
|
+
is_new_thread = False
|
|
182
|
+
if thread_id is None:
|
|
183
|
+
title = title_override or extract_title(all_lines)
|
|
184
|
+
if is_subagent:
|
|
185
|
+
title = f"🤖 {title}"
|
|
186
|
+
thread_id = create_thread(
|
|
187
|
+
session,
|
|
188
|
+
source=source,
|
|
189
|
+
source_id=source_id,
|
|
190
|
+
title=title,
|
|
191
|
+
thread_type="system" if is_subagent else "conversation",
|
|
192
|
+
description=extract_description(all_lines),
|
|
193
|
+
source_metadata=_cc_origin_metadata(all_lines, source_id, session=session),
|
|
194
|
+
)
|
|
195
|
+
is_new_thread = True
|
|
196
|
+
|
|
197
|
+
events_created, last_uuid = import_lines(
|
|
198
|
+
session, thread_id, new_lines, parser, builder,
|
|
199
|
+
source=source, source_id=source_id, cross_pass_dedup=merged_continuation,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
# A freshly-created thread that imported nothing (no importable content) is
|
|
203
|
+
# cleaned up so we don't leave an empty thread behind — row AND staged truth
|
|
204
|
+
# record, so no ghost threads/<id>.jsonl survives the commit.
|
|
205
|
+
if is_new_thread and events_created == 0:
|
|
206
|
+
discard_new_thread(session, thread_id)
|
|
207
|
+
thread_id = 0
|
|
208
|
+
is_new_thread = False
|
|
209
|
+
elif thread_id and events_created:
|
|
210
|
+
# Keep metadata current. New threads already carry title/description from
|
|
211
|
+
# create; an existing thread re-syncs a rename / AI title and backfills a
|
|
212
|
+
# missing description. Models are re-derived from the (now-written) events
|
|
213
|
+
# for both, so the sidebar model filter stays current.
|
|
214
|
+
if not is_new_thread:
|
|
215
|
+
refreshed = extract_session_title(all_lines)
|
|
216
|
+
if refreshed:
|
|
217
|
+
update_thread_title(session, thread_id, refreshed[:100])
|
|
218
|
+
thread = session.get(Thread, thread_id)
|
|
219
|
+
if thread is not None and not thread.description:
|
|
220
|
+
desc = extract_description(all_lines)
|
|
221
|
+
if desc:
|
|
222
|
+
update_thread_description(session, thread_id, desc)
|
|
223
|
+
set_thread_models_from_events(session, thread_id)
|
|
224
|
+
|
|
225
|
+
upsert_import_state(
|
|
226
|
+
session,
|
|
227
|
+
source=source,
|
|
228
|
+
source_id=source_id,
|
|
229
|
+
thread_id=thread_id or None,
|
|
230
|
+
last_line_count=total_lines,
|
|
231
|
+
last_file_size=current_file_size,
|
|
232
|
+
last_content_hash=cursor.content_hash,
|
|
233
|
+
last_message_uuid=last_uuid,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
return IncrementalImportResult(
|
|
237
|
+
lines_processed=len(new_lines),
|
|
238
|
+
events_created=events_created,
|
|
239
|
+
thread_id=thread_id or 0,
|
|
240
|
+
is_new_thread=is_new_thread,
|
|
241
|
+
last_message_uuid=last_uuid,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def import_session_incremental(
|
|
246
|
+
session_path,
|
|
247
|
+
source_id: str,
|
|
248
|
+
parser: Optional[ClaudeCodeParser] = None,
|
|
249
|
+
builder: Optional[DefaultEventBuilder] = None,
|
|
250
|
+
*,
|
|
251
|
+
source: str = SOURCE,
|
|
252
|
+
session=None,
|
|
253
|
+
) -> IncrementalImportResult:
|
|
254
|
+
"""Import a Claude Code JSONL transcript incrementally.
|
|
255
|
+
|
|
256
|
+
Reads the file, then runs the import. When ``session`` is given the caller owns
|
|
257
|
+
the transaction; otherwise the whole import commits atomically in a fresh
|
|
258
|
+
session (so events + watermark land together, or not at all).
|
|
259
|
+
|
|
260
|
+
``source`` is the thread/import-state source label. It defaults to
|
|
261
|
+
``"claude-code"`` and is rarely overridden — the one case is a *host store that
|
|
262
|
+
writes Claude-Code-shaped JSONL under a different identity* (e.g. cloth's CLI
|
|
263
|
+
transcripts), which a deployment watcher imports under its own source so the
|
|
264
|
+
threads carry the right provenance rather than masquerading as claude-code.
|
|
265
|
+
"""
|
|
266
|
+
session_path = Path(session_path)
|
|
267
|
+
if not session_path.exists():
|
|
268
|
+
raise FileNotFoundError(f"Session file not found: {session_path}")
|
|
269
|
+
|
|
270
|
+
parser = parser or ClaudeCodeParser()
|
|
271
|
+
builder = builder or DefaultEventBuilder()
|
|
272
|
+
source_bytes = read_source_bytes(session_path)
|
|
273
|
+
all_lines = parse_session_lines(source_bytes, session_path.name)
|
|
274
|
+
sidecar_lines = read_sidecar_lines(session_path)
|
|
275
|
+
|
|
276
|
+
def _run(s) -> IncrementalImportResult:
|
|
277
|
+
result = _import_cc(s, source_id, all_lines, source_bytes, parser, builder, source=source)
|
|
278
|
+
# The hook-context sidecar has its own line-count cursor, independent of the
|
|
279
|
+
# session file's size watermark — so a grown sidecar imports even when the
|
|
280
|
+
# main file is unchanged (and _import_cc returned early).
|
|
281
|
+
if result.thread_id and sidecar_lines:
|
|
282
|
+
n = import_sidecar_lines(s, result.thread_id, source_id, sidecar_lines)
|
|
283
|
+
if n:
|
|
284
|
+
result = replace(result, events_created=result.events_created + n)
|
|
285
|
+
return result
|
|
286
|
+
|
|
287
|
+
if session is not None:
|
|
288
|
+
return _run(session)
|
|
289
|
+
|
|
290
|
+
with get_session() as s:
|
|
291
|
+
result = _run(s)
|
|
292
|
+
s.commit()
|
|
293
|
+
return result
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""Claude Science (the Anthropic *AI Workbench* desktop app) session import.
|
|
2
|
+
|
|
3
|
+
Claude Science keeps each org's work in a single live SQLite DB at
|
|
4
|
+
``~/.claude-science/orgs/<org>/operon-cli.db``. A conversation is a **frame** (the
|
|
5
|
+
``frames`` table): a root frame (``parent_frame_id IS NULL``) is a top-level chat,
|
|
6
|
+
and a child frame is a subagent run (e.g. a ``REVIEWER`` spawned by the main
|
|
7
|
+
``OPERON`` agent). A frame's transcript is its ``frame_messages`` rows — one
|
|
8
|
+
Anthropic-shaped message per ``(frame_id, idx)`` in ``msg_json``
|
|
9
|
+
(``{"role", "content": [blocks]}`` where a block is text / thinking / tool_use /
|
|
10
|
+
tool_result / image).
|
|
11
|
+
|
|
12
|
+
We scan the DB the way the Cursor / OpenCode importers do (open read-only, walk the
|
|
13
|
+
conversations, incremental-skip via the import_state cursor), but a frame's messages
|
|
14
|
+
are *already* Claude-shaped — so rather than re-deriving the NormalizedMessage blocks
|
|
15
|
+
by hand, we synthesize Claude-Code-style JSONL lines from them and run them through
|
|
16
|
+
the standard CC import path under ``source="claude-science"`` (exactly as cowork does
|
|
17
|
+
for its ``audit.jsonl``). Continuation detection is CC-only, so it's skipped.
|
|
18
|
+
|
|
19
|
+
Two store quirks shape the mapping:
|
|
20
|
+
|
|
21
|
+
- **No per-message timestamp.** ``frame_messages`` carries none — only the frame has
|
|
22
|
+
``created_at`` / ``completed_at``. So we stamp each message a deterministic
|
|
23
|
+
``created_at + idx·STEP`` (see :func:`_synthesize_line`): monotonic by ``idx`` and
|
|
24
|
+
stable across incremental passes (``idx`` and ``created_at`` don't move), which is
|
|
25
|
+
what the event assembler needs. dedup is timestamp-free (content-keyed), so this
|
|
26
|
+
affects only ordering, never idempotence.
|
|
27
|
+
- **Subagent frames.** A child frame (it has a ``parent_frame_id``) imports as a
|
|
28
|
+
hidden ``thread_type="system"`` thread with a 🤖-prefixed title, mirroring how CC
|
|
29
|
+
Task-tool subagents are filed.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import json
|
|
35
|
+
import logging
|
|
36
|
+
import sqlite3
|
|
37
|
+
from dataclasses import dataclass, field
|
|
38
|
+
from datetime import datetime, timezone
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import Any, Optional
|
|
41
|
+
|
|
42
|
+
from thread_archive._thread_import import DefaultEventBuilder
|
|
43
|
+
from thread_archive._thread_import.parsers.claude_code import ClaudeCodeParser
|
|
44
|
+
|
|
45
|
+
from .._store import get_session
|
|
46
|
+
from ._events import import_lines
|
|
47
|
+
from ._state import (
|
|
48
|
+
adopt_if_unwatermarked,
|
|
49
|
+
create_thread,
|
|
50
|
+
discard_new_thread,
|
|
51
|
+
get_import_state,
|
|
52
|
+
get_thread_by_source,
|
|
53
|
+
set_thread_models_from_events,
|
|
54
|
+
upsert_import_state,
|
|
55
|
+
)
|
|
56
|
+
from ._titles import extract_title
|
|
57
|
+
|
|
58
|
+
logger = logging.getLogger(__name__)
|
|
59
|
+
|
|
60
|
+
SOURCE = "claude-science"
|
|
61
|
+
|
|
62
|
+
# Synthetic per-message spacing (ms). The store records no per-message time, so we
|
|
63
|
+
# space messages a second apart from the frame's start — monotonic and stable per
|
|
64
|
+
# ``idx``, which is all the assembler's ordering needs.
|
|
65
|
+
_STEP_MS = 1000
|
|
66
|
+
|
|
67
|
+
# Frames that aren't conversations: the per-project "User Uploads" container.
|
|
68
|
+
_NON_CONVERSATION_TYPES = ("uploads",)
|
|
69
|
+
|
|
70
|
+
# Anthropic ships a demo project ("Example project", id ``proj_example``) of pre-built
|
|
71
|
+
# analyses — scRNA-seq, CRISPR, phylogenetics — seeded into every install. Those are
|
|
72
|
+
# *Anthropic's* canned transcripts, not the user's own work, so they're kept out of
|
|
73
|
+
# the archive (importing them would inject fabricated "science you did" threads). The
|
|
74
|
+
# user's real frames live under their own ``proj_<hash>`` ids and import normally.
|
|
75
|
+
_SEEDED_PROJECT_IDS = ("proj_example",)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class ClaudeScienceImportResult:
|
|
80
|
+
events_created: int
|
|
81
|
+
thread_id: int
|
|
82
|
+
is_new_thread: bool
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class ClaudeScienceDbScanResult:
|
|
87
|
+
frames_processed: int
|
|
88
|
+
frames_imported: int
|
|
89
|
+
events_created: int
|
|
90
|
+
frames_failed: int = 0
|
|
91
|
+
errors: list[str] = field(default_factory=list)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _iso_from_ms(ms: int) -> str:
|
|
95
|
+
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _synthesize_line(frame_id: str, idx: int, msg: dict, model: Optional[str], base_ms: int) -> Optional[dict]:
|
|
99
|
+
"""Turn one ``frame_messages`` row into a Claude-Code-shaped JSONL line.
|
|
100
|
+
|
|
101
|
+
The message body (``role`` + ``content`` blocks) is already Anthropic-shaped, so
|
|
102
|
+
it drops straight into the CC line's ``message`` field; we only add the envelope
|
|
103
|
+
(``type`` / ``uuid`` / synthetic ``timestamp`` / ``sessionId``) the parser keys
|
|
104
|
+
off. A real ``_uuid`` is used when present so re-imports dedup on a stable id;
|
|
105
|
+
otherwise ``{frame_id}:{idx}`` is a stable synthetic stand-in."""
|
|
106
|
+
role = msg.get("role")
|
|
107
|
+
if role not in ("user", "assistant"):
|
|
108
|
+
return None
|
|
109
|
+
content = msg.get("content")
|
|
110
|
+
if content is None:
|
|
111
|
+
return None
|
|
112
|
+
message: dict[str, Any] = {"role": role, "content": content}
|
|
113
|
+
if role == "assistant" and model:
|
|
114
|
+
message["model"] = model
|
|
115
|
+
return {
|
|
116
|
+
"type": role,
|
|
117
|
+
"uuid": msg.get("_uuid") or f"{frame_id}:{idx}",
|
|
118
|
+
"timestamp": _iso_from_ms(base_ms + idx * _STEP_MS),
|
|
119
|
+
"sessionId": frame_id,
|
|
120
|
+
"message": message,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _frame_title(frame: dict, lines: list[dict]) -> str:
|
|
125
|
+
"""A thread title for a frame: its ``name`` (what the app shows), else its
|
|
126
|
+
``task_summary``, else the first-user-message fallback. A child (subagent) frame
|
|
127
|
+
gets a 🤖 prefix. Onboarding/system-prefixed first lines fall back to an
|
|
128
|
+
agent-named label rather than leaking the injected ``[System]`` block."""
|
|
129
|
+
title = (frame.get("name") or "").strip() or (frame.get("task_summary") or "").strip()
|
|
130
|
+
if not title:
|
|
131
|
+
derived = extract_title(lines)
|
|
132
|
+
if derived and derived != "Claude Code Session" and not derived.startswith("[System]"):
|
|
133
|
+
title = derived
|
|
134
|
+
else:
|
|
135
|
+
agent = (frame.get("agent_name") or "").strip().replace("_", " ").title()
|
|
136
|
+
title = f"{agent} Session" if agent else "Claude Science Session"
|
|
137
|
+
if frame.get("parent_frame_id"):
|
|
138
|
+
title = f"🤖 {title}"
|
|
139
|
+
return title[:100]
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _frame_metadata(org_uuid: str, frame: dict) -> dict:
|
|
143
|
+
"""Origin metadata for a Claude Science thread — the org, the frame's identity in
|
|
144
|
+
the app, and (for a subagent) the parent/root frame it was spawned under."""
|
|
145
|
+
meta: dict[str, Any] = {
|
|
146
|
+
"app": "claude-science",
|
|
147
|
+
"org_uuid": org_uuid,
|
|
148
|
+
"frame_id": frame.get("id"),
|
|
149
|
+
"agent_name": frame.get("agent_name"),
|
|
150
|
+
"conversation_type": frame.get("conversation_type"),
|
|
151
|
+
"project_id": frame.get("project_id"),
|
|
152
|
+
"model": frame.get("model"),
|
|
153
|
+
}
|
|
154
|
+
if frame.get("parent_frame_id"):
|
|
155
|
+
meta["is_subagent"] = True
|
|
156
|
+
meta["parent_frame_id"] = frame.get("parent_frame_id")
|
|
157
|
+
meta["root_frame_id"] = frame.get("root_frame_id")
|
|
158
|
+
return {k: v for k, v in meta.items() if v is not None}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _run_frame(
|
|
162
|
+
session,
|
|
163
|
+
org_uuid: str,
|
|
164
|
+
frame: dict,
|
|
165
|
+
message_rows: list,
|
|
166
|
+
parser: ClaudeCodeParser,
|
|
167
|
+
builder: DefaultEventBuilder,
|
|
168
|
+
) -> ClaudeScienceImportResult:
|
|
169
|
+
frame_id = frame["id"]
|
|
170
|
+
source_id = f"{org_uuid}:{frame_id}"
|
|
171
|
+
import_state = get_import_state(session, SOURCE, source_id)
|
|
172
|
+
|
|
173
|
+
total = len(message_rows)
|
|
174
|
+
start = import_state.last_line_count if import_state else 0
|
|
175
|
+
if import_state and start >= total:
|
|
176
|
+
return ClaudeScienceImportResult(0, import_state.thread_id or 0, False)
|
|
177
|
+
|
|
178
|
+
base_ms = int(frame.get("created_at") or 0)
|
|
179
|
+
model = frame.get("model")
|
|
180
|
+
lines: list[dict] = []
|
|
181
|
+
for row in message_rows:
|
|
182
|
+
idx, raw = row[0], row[1]
|
|
183
|
+
try:
|
|
184
|
+
msg = json.loads(raw)
|
|
185
|
+
except (json.JSONDecodeError, TypeError):
|
|
186
|
+
continue
|
|
187
|
+
if not isinstance(msg, dict):
|
|
188
|
+
continue
|
|
189
|
+
line = _synthesize_line(frame_id, idx, msg, model, base_ms)
|
|
190
|
+
if line is not None:
|
|
191
|
+
lines.append(line)
|
|
192
|
+
|
|
193
|
+
# Resolve the thread (watermark wins, else lookup by source) before any create.
|
|
194
|
+
thread_id: Optional[int] = (
|
|
195
|
+
import_state.thread_id if (import_state and import_state.thread_id) else None
|
|
196
|
+
)
|
|
197
|
+
if thread_id is None:
|
|
198
|
+
existing = get_thread_by_source(session, SOURCE, source_id)
|
|
199
|
+
thread_id = existing.id if existing else None
|
|
200
|
+
|
|
201
|
+
# Reindex safety: a thread with events but no watermark (cursor lost to a
|
|
202
|
+
# `rm index.db && reindex`) is adopted at EOF rather than re-imported from 0.
|
|
203
|
+
if import_state is None and adopt_if_unwatermarked(
|
|
204
|
+
session, source=SOURCE, source_id=source_id,
|
|
205
|
+
thread_id=thread_id, total_lines=total, file_size=0,
|
|
206
|
+
):
|
|
207
|
+
return ClaudeScienceImportResult(0, thread_id or 0, False)
|
|
208
|
+
|
|
209
|
+
is_new_thread = False
|
|
210
|
+
if thread_id is None:
|
|
211
|
+
thread_id = create_thread(
|
|
212
|
+
session,
|
|
213
|
+
source=SOURCE,
|
|
214
|
+
source_id=source_id,
|
|
215
|
+
title=_frame_title(frame, lines),
|
|
216
|
+
thread_type="system" if frame.get("parent_frame_id") else "conversation",
|
|
217
|
+
source_metadata=_frame_metadata(org_uuid, frame),
|
|
218
|
+
)
|
|
219
|
+
is_new_thread = True
|
|
220
|
+
|
|
221
|
+
new_lines = lines[start:]
|
|
222
|
+
events_created, last_uuid = import_lines(
|
|
223
|
+
session, thread_id, new_lines, parser, builder, source=SOURCE, source_id=source_id,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# A freshly-created thread that imported nothing (no importable content) is
|
|
227
|
+
# cleaned up so we don't leave an empty thread behind — row AND staged truth
|
|
228
|
+
# record, so no ghost threads/<id>.jsonl survives the commit.
|
|
229
|
+
if is_new_thread and events_created == 0:
|
|
230
|
+
discard_new_thread(session, thread_id)
|
|
231
|
+
thread_id = 0
|
|
232
|
+
is_new_thread = False
|
|
233
|
+
elif thread_id and events_created:
|
|
234
|
+
set_thread_models_from_events(session, thread_id)
|
|
235
|
+
|
|
236
|
+
upsert_import_state(
|
|
237
|
+
session,
|
|
238
|
+
source=SOURCE,
|
|
239
|
+
source_id=source_id,
|
|
240
|
+
thread_id=thread_id or None,
|
|
241
|
+
last_line_count=total,
|
|
242
|
+
last_file_size=0,
|
|
243
|
+
last_message_uuid=last_uuid,
|
|
244
|
+
)
|
|
245
|
+
return ClaudeScienceImportResult(events_created, thread_id or 0, is_new_thread)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def import_claude_science_frame(
|
|
249
|
+
org_uuid: str,
|
|
250
|
+
frame: dict,
|
|
251
|
+
message_rows: list,
|
|
252
|
+
parser: Optional[ClaudeCodeParser] = None,
|
|
253
|
+
builder: Optional[DefaultEventBuilder] = None,
|
|
254
|
+
*,
|
|
255
|
+
session=None,
|
|
256
|
+
) -> ClaudeScienceImportResult:
|
|
257
|
+
"""Import one frame's transcript (atomic per-frame transaction when no session)."""
|
|
258
|
+
parser = parser or ClaudeCodeParser()
|
|
259
|
+
builder = builder or DefaultEventBuilder()
|
|
260
|
+
if session is not None:
|
|
261
|
+
return _run_frame(session, org_uuid, frame, message_rows, parser, builder)
|
|
262
|
+
with get_session() as s:
|
|
263
|
+
result = _run_frame(s, org_uuid, frame, message_rows, parser, builder)
|
|
264
|
+
s.commit()
|
|
265
|
+
return result
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
# The frame columns the importer reads (title/metadata/timestamps + identity).
|
|
269
|
+
_FRAME_COLUMNS = (
|
|
270
|
+
"id", "parent_frame_id", "root_frame_id", "agent_name", "conversation_type",
|
|
271
|
+
"name", "task_summary", "model", "project_id", "created_at",
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def import_claude_science_db(db_path, org_uuid: str) -> ClaudeScienceDbScanResult:
|
|
276
|
+
"""Open a Claude Science ``operon-cli.db`` (read-only) and import every frame in
|
|
277
|
+
it under ``org_uuid``. The DB is live, so we open it ``mode=ro`` with a busy
|
|
278
|
+
timeout and never write to it."""
|
|
279
|
+
db_path = Path(db_path)
|
|
280
|
+
summary = ClaudeScienceDbScanResult(0, 0, 0)
|
|
281
|
+
parser = ClaudeCodeParser()
|
|
282
|
+
builder = DefaultEventBuilder()
|
|
283
|
+
|
|
284
|
+
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
|
285
|
+
try:
|
|
286
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
287
|
+
if not conn.execute(
|
|
288
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='frames'"
|
|
289
|
+
).fetchone():
|
|
290
|
+
return summary
|
|
291
|
+
|
|
292
|
+
type_ph = ", ".join("?" for _ in _NON_CONVERSATION_TYPES)
|
|
293
|
+
proj_ph = ", ".join("?" for _ in _SEEDED_PROJECT_IDS)
|
|
294
|
+
frames = conn.execute(
|
|
295
|
+
f"SELECT {', '.join(_FRAME_COLUMNS)} FROM frames "
|
|
296
|
+
f"WHERE conversation_type NOT IN ({type_ph}) "
|
|
297
|
+
f" AND (project_id IS NULL OR project_id NOT IN ({proj_ph})) "
|
|
298
|
+
f"ORDER BY created_at",
|
|
299
|
+
(*_NON_CONVERSATION_TYPES, *_SEEDED_PROJECT_IDS),
|
|
300
|
+
).fetchall()
|
|
301
|
+
|
|
302
|
+
for frame_row in frames:
|
|
303
|
+
frame = dict(zip(_FRAME_COLUMNS, frame_row))
|
|
304
|
+
frame_id = frame["id"]
|
|
305
|
+
message_rows = conn.execute(
|
|
306
|
+
"SELECT idx, msg_json FROM frame_messages WHERE frame_id = ? ORDER BY idx",
|
|
307
|
+
(frame_id,),
|
|
308
|
+
).fetchall()
|
|
309
|
+
if not message_rows:
|
|
310
|
+
continue
|
|
311
|
+
summary.frames_processed += 1
|
|
312
|
+
try:
|
|
313
|
+
result = import_claude_science_frame(
|
|
314
|
+
org_uuid, frame, message_rows, parser, builder
|
|
315
|
+
)
|
|
316
|
+
if result.events_created > 0:
|
|
317
|
+
summary.frames_imported += 1
|
|
318
|
+
summary.events_created += result.events_created
|
|
319
|
+
except Exception as e: # noqa: BLE001 — one bad frame must not stop the scan
|
|
320
|
+
# Counted out to the watcher, not just logged — see the same guard in
|
|
321
|
+
# opencode: a silently-skipped frame reads as "nothing new" and leaves
|
|
322
|
+
# `archive status` green while a conversation is missing.
|
|
323
|
+
logger.exception(
|
|
324
|
+
"import_claude_science_db: frame %s failed; skipping", frame_id[:8]
|
|
325
|
+
)
|
|
326
|
+
summary.frames_failed += 1
|
|
327
|
+
summary.errors.append(f"frame {frame_id[:8]}: {e}")
|
|
328
|
+
finally:
|
|
329
|
+
conn.close()
|
|
330
|
+
return summary
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""cloth session import — cloth is a Claude-Code-shaped terminal harness.
|
|
2
|
+
|
|
3
|
+
cloth writes the same per-session JSONL shape as Claude Code (``user`` / ``assistant``
|
|
4
|
+
lines, plus a ``cloth_meta`` line the parser harmlessly ignores), so its importer
|
|
5
|
+
**delegates to the claude_code line-stream import** under ``source="cloth"`` rather
|
|
6
|
+
than duplicating a parser. Threads land with ``source="cloth"`` and ``source_id`` set
|
|
7
|
+
to the bare session-file stem (the CLI's session uuid — already globally unique — so
|
|
8
|
+
no prefix; see ``watcher.sources.cloth_watcher``). Idempotence, the truth-log seam, and
|
|
9
|
+
incremental watermarking all come from the shared claude_code path unchanged.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from ._result import IncrementalImportResult
|
|
15
|
+
from .claude_code import import_session_incremental
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def import_cloth_session_incremental(session_path, source_id: str, *, session=None) -> IncrementalImportResult:
|
|
19
|
+
"""Import one cloth CLI transcript into the event log, under ``source="cloth"``."""
|
|
20
|
+
return import_session_incremental(session_path, source_id, source="cloth", session=session)
|