thread-archive 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. thread_archive/__init__.py +34 -0
  2. thread_archive/_api.py +2235 -0
  3. thread_archive/_config.py +126 -0
  4. thread_archive/_importers/__init__.py +81 -0
  5. thread_archive/_importers/_continuation.py +136 -0
  6. thread_archive/_importers/_cursor.py +103 -0
  7. thread_archive/_importers/_events.py +244 -0
  8. thread_archive/_importers/_line_stream.py +193 -0
  9. thread_archive/_importers/_read.py +82 -0
  10. thread_archive/_importers/_result.py +17 -0
  11. thread_archive/_importers/_sidecar.py +113 -0
  12. thread_archive/_importers/_state.py +240 -0
  13. thread_archive/_importers/_titles.py +179 -0
  14. thread_archive/_importers/antigravity.py +254 -0
  15. thread_archive/_importers/claude_code.py +293 -0
  16. thread_archive/_importers/claude_science.py +330 -0
  17. thread_archive/_importers/cloth.py +20 -0
  18. thread_archive/_importers/codex.py +324 -0
  19. thread_archive/_importers/cowork.py +107 -0
  20. thread_archive/_importers/cursor.py +432 -0
  21. thread_archive/_importers/exports.py +389 -0
  22. thread_archive/_importers/grok.py +600 -0
  23. thread_archive/_importers/opencode.py +538 -0
  24. thread_archive/_knowledge/__init__.py +67 -0
  25. thread_archive/_knowledge/_claims.py +116 -0
  26. thread_archive/_knowledge/_community.py +75 -0
  27. thread_archive/_knowledge/graph.py +208 -0
  28. thread_archive/_knowledge/materialize.py +212 -0
  29. thread_archive/_knowledge/write.py +438 -0
  30. thread_archive/_launchd.py +167 -0
  31. thread_archive/_mcp/__init__.py +1 -0
  32. thread_archive/_mcp/librarian.py +151 -0
  33. thread_archive/_mcp/server.py +307 -0
  34. thread_archive/_retrieval/__init__.py +280 -0
  35. thread_archive/_retrieval/_classify.py +80 -0
  36. thread_archive/_retrieval/_codex.py +157 -0
  37. thread_archive/_retrieval/_context.py +123 -0
  38. thread_archive/_retrieval/_extract.py +184 -0
  39. thread_archive/_retrieval/embed.py +186 -0
  40. thread_archive/_retrieval/format.py +149 -0
  41. thread_archive/_retrieval/fts.py +538 -0
  42. thread_archive/_retrieval/rank.py +228 -0
  43. thread_archive/_retrieval/read.py +908 -0
  44. thread_archive/_retrieval/rerank.py +143 -0
  45. thread_archive/_retrieval/vectors.py +611 -0
  46. thread_archive/_scripts/__init__.py +1 -0
  47. thread_archive/_scripts/backfill_recompute.py +328 -0
  48. thread_archive/_scripts/backfill_reconcile.py +296 -0
  49. thread_archive/_scripts/denamespace_dedup_keys.py +120 -0
  50. thread_archive/_scripts/recover_dropped_events.py +190 -0
  51. thread_archive/_scripts/repair_grok_tool_names.py +118 -0
  52. thread_archive/_scripts/repair_grok_tool_names_backup_20260704T155625Z.json +275 -0
  53. thread_archive/_scripts/repair_grok_tool_names_plan_20260704.json +435 -0
  54. thread_archive/_setup/__init__.py +12 -0
  55. thread_archive/_setup/clients.py +89 -0
  56. thread_archive/_setup/wizard.py +474 -0
  57. thread_archive/_store/__init__.py +45 -0
  58. thread_archive/_store/_base.py +173 -0
  59. thread_archive/_store/_defaults.py +57 -0
  60. thread_archive/_store/_types.py +38 -0
  61. thread_archive/_store/models.py +370 -0
  62. thread_archive/_store/schema.py +84 -0
  63. thread_archive/_thread_import/__init__.py +40 -0
  64. thread_archive/_thread_import/api.py +78 -0
  65. thread_archive/_thread_import/event_builder.py +810 -0
  66. thread_archive/_thread_import/exporters/__init__.py +9 -0
  67. thread_archive/_thread_import/exporters/_cursor_kv_mixin.py +166 -0
  68. thread_archive/_thread_import/exporters/_cursor_parse_mixin.py +149 -0
  69. thread_archive/_thread_import/exporters/cursor.py +582 -0
  70. thread_archive/_thread_import/exporters/cursor_parse.py +145 -0
  71. thread_archive/_thread_import/parsers/__init__.py +191 -0
  72. thread_archive/_thread_import/parsers/base.py +698 -0
  73. thread_archive/_thread_import/parsers/chatgpt.py +722 -0
  74. thread_archive/_thread_import/parsers/chatgpt_content.py +332 -0
  75. thread_archive/_thread_import/parsers/claude.py +443 -0
  76. thread_archive/_thread_import/parsers/claude_code.py +1035 -0
  77. thread_archive/_thread_import/parsers/claude_code_blocks.py +415 -0
  78. thread_archive/_thread_import/parsers/claude_code_ide.py +122 -0
  79. thread_archive/_thread_import/parsers/claude_code_sessions.py +90 -0
  80. thread_archive/_thread_import/parsers/config/__init__.py +26 -0
  81. thread_archive/_thread_import/parsers/config/base.py +220 -0
  82. thread_archive/_thread_import/parsers/cursor.py +538 -0
  83. thread_archive/_thread_import/parsers/cursor_blocks.py +365 -0
  84. thread_archive/_thread_import/parsers/pipeline/__init__.py +29 -0
  85. thread_archive/_thread_import/parsers/pipeline/base.py +251 -0
  86. thread_archive/_thread_import/parsers/pipeline/interfaces.py +191 -0
  87. thread_archive/_thread_import/parsers/transformers/__init__.py +22 -0
  88. thread_archive/_thread_import/parsers/transformers/active_path.py +87 -0
  89. thread_archive/_thread_import/parsers/transformers/coalescing.py +389 -0
  90. thread_archive/_thread_import/parsers/transformers/ide_context.py +158 -0
  91. thread_archive/_thread_import/parsers/transformers/thinking_merge.py +68 -0
  92. thread_archive/_thread_import/parsers/types/__init__.py +56 -0
  93. thread_archive/_thread_import/parsers/types/chatgpt.py +99 -0
  94. thread_archive/_thread_import/parsers/types/claude.py +120 -0
  95. thread_archive/_thread_import/parsers/types/claude_code.py +139 -0
  96. thread_archive/_thread_import/parsers/types/cursor.py +109 -0
  97. thread_archive/_thread_import/parsers/validators/__init__.py +83 -0
  98. thread_archive/_thread_import/parsers/validators/base.py +168 -0
  99. thread_archive/_thread_import/parsers/validators/content.py +80 -0
  100. thread_archive/_thread_import/parsers/validators/referential.py +57 -0
  101. thread_archive/_thread_import/parsers/validators/thinking.py +143 -0
  102. thread_archive/_thread_import/parsers/validators/types.py +87 -0
  103. thread_archive/_thread_import/schemas/__init__.py +231 -0
  104. thread_archive/_thread_import/schemas/chatgpt/v1.json +197 -0
  105. thread_archive/_thread_import/schemas/chatgpt/v2.json +203 -0
  106. thread_archive/_thread_import/schemas/claude/v1.json +188 -0
  107. thread_archive/_thread_import/schemas/claude_code/v1.json +183 -0
  108. thread_archive/_thread_import/schemas/cursor/v1.json +143 -0
  109. thread_archive/_thread_import/schemas/cursor/v2.json +200 -0
  110. thread_archive/_thread_import/timestamps.py +57 -0
  111. thread_archive/_thread_import/tool_names.py +48 -0
  112. thread_archive/_truth/__init__.py +40 -0
  113. thread_archive/_truth/jsonl_log.py +2340 -0
  114. thread_archive/_truth/repair.py +322 -0
  115. thread_archive/_watcher/__init__.py +57 -0
  116. thread_archive/_watcher/base.py +65 -0
  117. thread_archive/_watcher/daemon.py +289 -0
  118. thread_archive/_watcher/export_drop.py +203 -0
  119. thread_archive/_watcher/exthost.py +316 -0
  120. thread_archive/_watcher/lazy.py +117 -0
  121. thread_archive/_watcher/sources.py +616 -0
  122. thread_archive/_web/__init__.py +15 -0
  123. thread_archive/_web/server.py +299 -0
  124. thread_archive/_web/static/assets/index-DECSH1Mz.css +10 -0
  125. thread_archive/_web/static/assets/index-Djq0FK0y.js +101 -0
  126. thread_archive/_web/static/index.html +13 -0
  127. thread_archive/cli.py +746 -0
  128. thread_archive-0.0.2.dist-info/METADATA +296 -0
  129. thread_archive-0.0.2.dist-info/RECORD +132 -0
  130. thread_archive-0.0.2.dist-info/WHEEL +4 -0
  131. thread_archive-0.0.2.dist-info/entry_points.txt +6 -0
  132. thread_archive-0.0.2.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,293 @@
1
+ """Claude Code incremental import.
2
+
3
+ Reduced to the conversation-archive essentials and run as a single atomic
4
+ transaction. The watermark logic lives in :mod:`._cursor`: a content digest proves the
5
+ file was only appended to, and the line cursor (``new_lines = all_lines[start_line:]``)
6
+ then feeds just the grown tail into the importer — a source rewritten under the cursor
7
+ rewinds and re-imports whole instead. Idempotence across re-reads (and across a
8
+ rewind) is guaranteed by the dedup_key membership check in :func:`import_lines`.
9
+
10
+ **Atomicity:** the whole import — thread create, events, watermark — commits in one
11
+ session, so "the watermark advances only after a successful import" holds by
12
+ construction.
13
+
14
+ Deferred (enhancements, not core): compaction continuation/fork detection, subagent
15
+ session handling, sidecar hook context, batch-thread adoption, description/models
16
+ backfill, and rich title extraction.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import logging
22
+ from dataclasses import replace
23
+ from pathlib import Path
24
+ from typing import Any, Optional
25
+
26
+ from thread_archive._thread_import import DefaultEventBuilder
27
+ from thread_archive._thread_import.parsers.claude_code import ClaudeCodeParser
28
+
29
+ from .._store import Thread, get_session
30
+ from ._continuation import resolve_continuation_thread
31
+ from ._cursor import resolve_source_cursor
32
+ from ._events import import_lines
33
+ from ._read import parse_session_lines, read_source_bytes
34
+ from ._result import IncrementalImportResult
35
+ from ._sidecar import import_sidecar_lines, read_sidecar_lines
36
+ from ._state import (
37
+ adopt_if_unwatermarked,
38
+ create_thread,
39
+ discard_new_thread,
40
+ get_import_state,
41
+ get_thread_by_source,
42
+ lookup_parent_thread,
43
+ set_thread_models_from_events,
44
+ update_thread_description,
45
+ update_thread_title,
46
+ upsert_import_state,
47
+ )
48
+ from ._titles import extract_description, extract_session_title, extract_title
49
+
50
+ logger = logging.getLogger(__name__)
51
+
52
+ SOURCE = "claude-code"
53
+
54
+
55
+ def _is_subagent_source_id(source_id: str) -> bool:
56
+ """True when this CC source_id names a Task-tool subagent transcript.
57
+
58
+ The watcher hands subagent files (``<project>/<uuid>/subagents/agent-*.jsonl``)
59
+ the same ``{project}:{stem}`` source_id as any other session; the stem keeps
60
+ Claude Code's ``agent-`` prefix, which is the marker that survives into the
61
+ import path. Subagent runs file as hidden ``thread_type='system'`` threads —
62
+ captured and searchable, but kept out of the archive sidebar."""
63
+ return source_id.rsplit(":", 1)[-1].startswith("agent-")
64
+
65
+
66
+ def _cc_origin_metadata(all_lines: list[dict], source_id: str, *, session=None) -> Optional[dict]:
67
+ """Origin-project metadata for a CC thread: the real ``cwd`` (first line that
68
+ carries one) + the munged ``project_dir`` from the source_id.
69
+
70
+ For a subagent transcript also stamp ``is_subagent`` plus the spawning session
71
+ (``parent_session_id`` from the lines' ``sessionId``) and the subagent's own
72
+ ``agent_id`` (from ``agentId``), so the run can be joined back to its parent
73
+ thread. When a ``session`` is supplied and the parent resolves to no thread, the
74
+ gap is logged rather than lost silently."""
75
+ meta: dict[str, Any] = {}
76
+ for line in all_lines:
77
+ cwd = line.get("cwd")
78
+ if isinstance(cwd, str) and cwd:
79
+ meta["cwd"] = cwd
80
+ break
81
+ if ":" in source_id:
82
+ meta["project_dir"] = source_id.rsplit(":", 1)[0]
83
+ if _is_subagent_source_id(source_id):
84
+ meta["is_subagent"] = True
85
+ for line in all_lines:
86
+ if "agent_id" not in meta and line.get("agentId"):
87
+ meta["agent_id"] = line["agentId"]
88
+ if "parent_session_id" not in meta and line.get("sessionId"):
89
+ meta["parent_session_id"] = line["sessionId"]
90
+ if "agent_id" in meta and "parent_session_id" in meta:
91
+ break
92
+ parent_sid = meta.get("parent_session_id")
93
+ project_dir = meta.get("project_dir")
94
+ if session is not None and parent_sid and project_dir:
95
+ parent_source_id = f"{project_dir}:{parent_sid}"
96
+ if lookup_parent_thread(session, parent_source_id) is None:
97
+ logger.warning(
98
+ "Subagent %s: parent_session_id %s resolves to no thread (%s) — "
99
+ "run can't be joined back to its parent",
100
+ source_id, parent_sid, parent_source_id,
101
+ )
102
+ return meta or None
103
+
104
+
105
+ def _import_cc(
106
+ session,
107
+ source_id: str,
108
+ all_lines: list[dict],
109
+ source_bytes: bytes,
110
+ parser: ClaudeCodeParser,
111
+ builder: DefaultEventBuilder,
112
+ *,
113
+ source: str = SOURCE,
114
+ title_override: Optional[str] = None,
115
+ ) -> IncrementalImportResult:
116
+ import_state = get_import_state(session, source, source_id)
117
+ cursor = resolve_source_cursor(import_state, source_bytes, source=source, source_id=source_id)
118
+ current_file_size = len(source_bytes)
119
+
120
+ if cursor.unchanged and import_state:
121
+ # Byte-identical to the last import. Stamp the digest if this watermark predates
122
+ # it, so the next poll's append proof has something to check against.
123
+ import_state.last_content_hash = cursor.content_hash
124
+ return IncrementalImportResult(
125
+ lines_processed=0,
126
+ events_created=0,
127
+ thread_id=import_state.thread_id or 0,
128
+ is_new_thread=False,
129
+ last_message_uuid=import_state.last_message_uuid,
130
+ )
131
+
132
+ total_lines = len(all_lines)
133
+ start_line = cursor.start_line
134
+ if start_line >= total_lines:
135
+ # The file changed but yielded no new parsed lines (a torn tail line the writer
136
+ # hasn't finished; a rewrite down to nothing). Advance size + digest so the
137
+ # completed line still reads as an append next poll — the line cursor tracks
138
+ # what actually parsed, so the line itself imports then.
139
+ if import_state:
140
+ import_state.last_line_count = total_lines
141
+ import_state.last_file_size = current_file_size
142
+ import_state.last_content_hash = cursor.content_hash
143
+ return IncrementalImportResult(
144
+ lines_processed=0,
145
+ events_created=0,
146
+ thread_id=(import_state.thread_id or 0) if import_state else 0,
147
+ is_new_thread=False,
148
+ last_message_uuid=import_state.last_message_uuid if import_state else None,
149
+ )
150
+
151
+ new_lines = all_lines[start_line:]
152
+
153
+ # Resolve the thread: the watermark's thread_id wins, else lookup by source.
154
+ thread_id: Optional[int] = (
155
+ import_state.thread_id if (import_state and import_state.thread_id) else None
156
+ )
157
+ if thread_id is None:
158
+ existing = get_thread_by_source(session, source, source_id)
159
+ thread_id = existing.id if existing else None
160
+
161
+ if import_state is None and adopt_if_unwatermarked(
162
+ session, source=source, source_id=source_id,
163
+ thread_id=thread_id, total_lines=total_lines, file_size=current_file_size,
164
+ content_hash=cursor.content_hash,
165
+ ):
166
+ return IncrementalImportResult(0, 0, thread_id or 0, False, None)
167
+
168
+ is_subagent = _is_subagent_source_id(source_id)
169
+
170
+ # Continuation / fork merge: a fresh CC session that continues an existing thread
171
+ # (created on context compaction) merges into it instead of fragmenting into its
172
+ # own thread. Subagents are independent transcripts; cloth-shaped sources (source
173
+ # != "claude-code") don't carry CC's compaction markers, so both skip it.
174
+ merged_continuation = False
175
+ if thread_id is None and source == SOURCE and not is_subagent:
176
+ parent_id = resolve_continuation_thread(session, all_lines, source_id, parser, builder)
177
+ if parent_id is not None:
178
+ thread_id = parent_id
179
+ merged_continuation = True
180
+
181
+ is_new_thread = False
182
+ if thread_id is None:
183
+ title = title_override or extract_title(all_lines)
184
+ if is_subagent:
185
+ title = f"🤖 {title}"
186
+ thread_id = create_thread(
187
+ session,
188
+ source=source,
189
+ source_id=source_id,
190
+ title=title,
191
+ thread_type="system" if is_subagent else "conversation",
192
+ description=extract_description(all_lines),
193
+ source_metadata=_cc_origin_metadata(all_lines, source_id, session=session),
194
+ )
195
+ is_new_thread = True
196
+
197
+ events_created, last_uuid = import_lines(
198
+ session, thread_id, new_lines, parser, builder,
199
+ source=source, source_id=source_id, cross_pass_dedup=merged_continuation,
200
+ )
201
+
202
+ # A freshly-created thread that imported nothing (no importable content) is
203
+ # cleaned up so we don't leave an empty thread behind — row AND staged truth
204
+ # record, so no ghost threads/<id>.jsonl survives the commit.
205
+ if is_new_thread and events_created == 0:
206
+ discard_new_thread(session, thread_id)
207
+ thread_id = 0
208
+ is_new_thread = False
209
+ elif thread_id and events_created:
210
+ # Keep metadata current. New threads already carry title/description from
211
+ # create; an existing thread re-syncs a rename / AI title and backfills a
212
+ # missing description. Models are re-derived from the (now-written) events
213
+ # for both, so the sidebar model filter stays current.
214
+ if not is_new_thread:
215
+ refreshed = extract_session_title(all_lines)
216
+ if refreshed:
217
+ update_thread_title(session, thread_id, refreshed[:100])
218
+ thread = session.get(Thread, thread_id)
219
+ if thread is not None and not thread.description:
220
+ desc = extract_description(all_lines)
221
+ if desc:
222
+ update_thread_description(session, thread_id, desc)
223
+ set_thread_models_from_events(session, thread_id)
224
+
225
+ upsert_import_state(
226
+ session,
227
+ source=source,
228
+ source_id=source_id,
229
+ thread_id=thread_id or None,
230
+ last_line_count=total_lines,
231
+ last_file_size=current_file_size,
232
+ last_content_hash=cursor.content_hash,
233
+ last_message_uuid=last_uuid,
234
+ )
235
+
236
+ return IncrementalImportResult(
237
+ lines_processed=len(new_lines),
238
+ events_created=events_created,
239
+ thread_id=thread_id or 0,
240
+ is_new_thread=is_new_thread,
241
+ last_message_uuid=last_uuid,
242
+ )
243
+
244
+
245
+ def import_session_incremental(
246
+ session_path,
247
+ source_id: str,
248
+ parser: Optional[ClaudeCodeParser] = None,
249
+ builder: Optional[DefaultEventBuilder] = None,
250
+ *,
251
+ source: str = SOURCE,
252
+ session=None,
253
+ ) -> IncrementalImportResult:
254
+ """Import a Claude Code JSONL transcript incrementally.
255
+
256
+ Reads the file, then runs the import. When ``session`` is given the caller owns
257
+ the transaction; otherwise the whole import commits atomically in a fresh
258
+ session (so events + watermark land together, or not at all).
259
+
260
+ ``source`` is the thread/import-state source label. It defaults to
261
+ ``"claude-code"`` and is rarely overridden — the one case is a *host store that
262
+ writes Claude-Code-shaped JSONL under a different identity* (e.g. cloth's CLI
263
+ transcripts), which a deployment watcher imports under its own source so the
264
+ threads carry the right provenance rather than masquerading as claude-code.
265
+ """
266
+ session_path = Path(session_path)
267
+ if not session_path.exists():
268
+ raise FileNotFoundError(f"Session file not found: {session_path}")
269
+
270
+ parser = parser or ClaudeCodeParser()
271
+ builder = builder or DefaultEventBuilder()
272
+ source_bytes = read_source_bytes(session_path)
273
+ all_lines = parse_session_lines(source_bytes, session_path.name)
274
+ sidecar_lines = read_sidecar_lines(session_path)
275
+
276
+ def _run(s) -> IncrementalImportResult:
277
+ result = _import_cc(s, source_id, all_lines, source_bytes, parser, builder, source=source)
278
+ # The hook-context sidecar has its own line-count cursor, independent of the
279
+ # session file's size watermark — so a grown sidecar imports even when the
280
+ # main file is unchanged (and _import_cc returned early).
281
+ if result.thread_id and sidecar_lines:
282
+ n = import_sidecar_lines(s, result.thread_id, source_id, sidecar_lines)
283
+ if n:
284
+ result = replace(result, events_created=result.events_created + n)
285
+ return result
286
+
287
+ if session is not None:
288
+ return _run(session)
289
+
290
+ with get_session() as s:
291
+ result = _run(s)
292
+ s.commit()
293
+ return result
@@ -0,0 +1,330 @@
1
+ """Claude Science (the Anthropic *AI Workbench* desktop app) session import.
2
+
3
+ Claude Science keeps each org's work in a single live SQLite DB at
4
+ ``~/.claude-science/orgs/<org>/operon-cli.db``. A conversation is a **frame** (the
5
+ ``frames`` table): a root frame (``parent_frame_id IS NULL``) is a top-level chat,
6
+ and a child frame is a subagent run (e.g. a ``REVIEWER`` spawned by the main
7
+ ``OPERON`` agent). A frame's transcript is its ``frame_messages`` rows — one
8
+ Anthropic-shaped message per ``(frame_id, idx)`` in ``msg_json``
9
+ (``{"role", "content": [blocks]}`` where a block is text / thinking / tool_use /
10
+ tool_result / image).
11
+
12
+ We scan the DB the way the Cursor / OpenCode importers do (open read-only, walk the
13
+ conversations, incremental-skip via the import_state cursor), but a frame's messages
14
+ are *already* Claude-shaped — so rather than re-deriving the NormalizedMessage blocks
15
+ by hand, we synthesize Claude-Code-style JSONL lines from them and run them through
16
+ the standard CC import path under ``source="claude-science"`` (exactly as cowork does
17
+ for its ``audit.jsonl``). Continuation detection is CC-only, so it's skipped.
18
+
19
+ Two store quirks shape the mapping:
20
+
21
+ - **No per-message timestamp.** ``frame_messages`` carries none — only the frame has
22
+ ``created_at`` / ``completed_at``. So we stamp each message a deterministic
23
+ ``created_at + idx·STEP`` (see :func:`_synthesize_line`): monotonic by ``idx`` and
24
+ stable across incremental passes (``idx`` and ``created_at`` don't move), which is
25
+ what the event assembler needs. dedup is timestamp-free (content-keyed), so this
26
+ affects only ordering, never idempotence.
27
+ - **Subagent frames.** A child frame (it has a ``parent_frame_id``) imports as a
28
+ hidden ``thread_type="system"`` thread with a 🤖-prefixed title, mirroring how CC
29
+ Task-tool subagents are filed.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import json
35
+ import logging
36
+ import sqlite3
37
+ from dataclasses import dataclass, field
38
+ from datetime import datetime, timezone
39
+ from pathlib import Path
40
+ from typing import Any, Optional
41
+
42
+ from thread_archive._thread_import import DefaultEventBuilder
43
+ from thread_archive._thread_import.parsers.claude_code import ClaudeCodeParser
44
+
45
+ from .._store import get_session
46
+ from ._events import import_lines
47
+ from ._state import (
48
+ adopt_if_unwatermarked,
49
+ create_thread,
50
+ discard_new_thread,
51
+ get_import_state,
52
+ get_thread_by_source,
53
+ set_thread_models_from_events,
54
+ upsert_import_state,
55
+ )
56
+ from ._titles import extract_title
57
+
58
+ logger = logging.getLogger(__name__)
59
+
60
+ SOURCE = "claude-science"
61
+
62
+ # Synthetic per-message spacing (ms). The store records no per-message time, so we
63
+ # space messages a second apart from the frame's start — monotonic and stable per
64
+ # ``idx``, which is all the assembler's ordering needs.
65
+ _STEP_MS = 1000
66
+
67
+ # Frames that aren't conversations: the per-project "User Uploads" container.
68
+ _NON_CONVERSATION_TYPES = ("uploads",)
69
+
70
+ # Anthropic ships a demo project ("Example project", id ``proj_example``) of pre-built
71
+ # analyses — scRNA-seq, CRISPR, phylogenetics — seeded into every install. Those are
72
+ # *Anthropic's* canned transcripts, not the user's own work, so they're kept out of
73
+ # the archive (importing them would inject fabricated "science you did" threads). The
74
+ # user's real frames live under their own ``proj_<hash>`` ids and import normally.
75
+ _SEEDED_PROJECT_IDS = ("proj_example",)
76
+
77
+
78
+ @dataclass
79
+ class ClaudeScienceImportResult:
80
+ events_created: int
81
+ thread_id: int
82
+ is_new_thread: bool
83
+
84
+
85
+ @dataclass
86
+ class ClaudeScienceDbScanResult:
87
+ frames_processed: int
88
+ frames_imported: int
89
+ events_created: int
90
+ frames_failed: int = 0
91
+ errors: list[str] = field(default_factory=list)
92
+
93
+
94
+ def _iso_from_ms(ms: int) -> str:
95
+ return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
96
+
97
+
98
+ def _synthesize_line(frame_id: str, idx: int, msg: dict, model: Optional[str], base_ms: int) -> Optional[dict]:
99
+ """Turn one ``frame_messages`` row into a Claude-Code-shaped JSONL line.
100
+
101
+ The message body (``role`` + ``content`` blocks) is already Anthropic-shaped, so
102
+ it drops straight into the CC line's ``message`` field; we only add the envelope
103
+ (``type`` / ``uuid`` / synthetic ``timestamp`` / ``sessionId``) the parser keys
104
+ off. A real ``_uuid`` is used when present so re-imports dedup on a stable id;
105
+ otherwise ``{frame_id}:{idx}`` is a stable synthetic stand-in."""
106
+ role = msg.get("role")
107
+ if role not in ("user", "assistant"):
108
+ return None
109
+ content = msg.get("content")
110
+ if content is None:
111
+ return None
112
+ message: dict[str, Any] = {"role": role, "content": content}
113
+ if role == "assistant" and model:
114
+ message["model"] = model
115
+ return {
116
+ "type": role,
117
+ "uuid": msg.get("_uuid") or f"{frame_id}:{idx}",
118
+ "timestamp": _iso_from_ms(base_ms + idx * _STEP_MS),
119
+ "sessionId": frame_id,
120
+ "message": message,
121
+ }
122
+
123
+
124
+ def _frame_title(frame: dict, lines: list[dict]) -> str:
125
+ """A thread title for a frame: its ``name`` (what the app shows), else its
126
+ ``task_summary``, else the first-user-message fallback. A child (subagent) frame
127
+ gets a 🤖 prefix. Onboarding/system-prefixed first lines fall back to an
128
+ agent-named label rather than leaking the injected ``[System]`` block."""
129
+ title = (frame.get("name") or "").strip() or (frame.get("task_summary") or "").strip()
130
+ if not title:
131
+ derived = extract_title(lines)
132
+ if derived and derived != "Claude Code Session" and not derived.startswith("[System]"):
133
+ title = derived
134
+ else:
135
+ agent = (frame.get("agent_name") or "").strip().replace("_", " ").title()
136
+ title = f"{agent} Session" if agent else "Claude Science Session"
137
+ if frame.get("parent_frame_id"):
138
+ title = f"🤖 {title}"
139
+ return title[:100]
140
+
141
+
142
+ def _frame_metadata(org_uuid: str, frame: dict) -> dict:
143
+ """Origin metadata for a Claude Science thread — the org, the frame's identity in
144
+ the app, and (for a subagent) the parent/root frame it was spawned under."""
145
+ meta: dict[str, Any] = {
146
+ "app": "claude-science",
147
+ "org_uuid": org_uuid,
148
+ "frame_id": frame.get("id"),
149
+ "agent_name": frame.get("agent_name"),
150
+ "conversation_type": frame.get("conversation_type"),
151
+ "project_id": frame.get("project_id"),
152
+ "model": frame.get("model"),
153
+ }
154
+ if frame.get("parent_frame_id"):
155
+ meta["is_subagent"] = True
156
+ meta["parent_frame_id"] = frame.get("parent_frame_id")
157
+ meta["root_frame_id"] = frame.get("root_frame_id")
158
+ return {k: v for k, v in meta.items() if v is not None}
159
+
160
+
161
+ def _run_frame(
162
+ session,
163
+ org_uuid: str,
164
+ frame: dict,
165
+ message_rows: list,
166
+ parser: ClaudeCodeParser,
167
+ builder: DefaultEventBuilder,
168
+ ) -> ClaudeScienceImportResult:
169
+ frame_id = frame["id"]
170
+ source_id = f"{org_uuid}:{frame_id}"
171
+ import_state = get_import_state(session, SOURCE, source_id)
172
+
173
+ total = len(message_rows)
174
+ start = import_state.last_line_count if import_state else 0
175
+ if import_state and start >= total:
176
+ return ClaudeScienceImportResult(0, import_state.thread_id or 0, False)
177
+
178
+ base_ms = int(frame.get("created_at") or 0)
179
+ model = frame.get("model")
180
+ lines: list[dict] = []
181
+ for row in message_rows:
182
+ idx, raw = row[0], row[1]
183
+ try:
184
+ msg = json.loads(raw)
185
+ except (json.JSONDecodeError, TypeError):
186
+ continue
187
+ if not isinstance(msg, dict):
188
+ continue
189
+ line = _synthesize_line(frame_id, idx, msg, model, base_ms)
190
+ if line is not None:
191
+ lines.append(line)
192
+
193
+ # Resolve the thread (watermark wins, else lookup by source) before any create.
194
+ thread_id: Optional[int] = (
195
+ import_state.thread_id if (import_state and import_state.thread_id) else None
196
+ )
197
+ if thread_id is None:
198
+ existing = get_thread_by_source(session, SOURCE, source_id)
199
+ thread_id = existing.id if existing else None
200
+
201
+ # Reindex safety: a thread with events but no watermark (cursor lost to a
202
+ # `rm index.db && reindex`) is adopted at EOF rather than re-imported from 0.
203
+ if import_state is None and adopt_if_unwatermarked(
204
+ session, source=SOURCE, source_id=source_id,
205
+ thread_id=thread_id, total_lines=total, file_size=0,
206
+ ):
207
+ return ClaudeScienceImportResult(0, thread_id or 0, False)
208
+
209
+ is_new_thread = False
210
+ if thread_id is None:
211
+ thread_id = create_thread(
212
+ session,
213
+ source=SOURCE,
214
+ source_id=source_id,
215
+ title=_frame_title(frame, lines),
216
+ thread_type="system" if frame.get("parent_frame_id") else "conversation",
217
+ source_metadata=_frame_metadata(org_uuid, frame),
218
+ )
219
+ is_new_thread = True
220
+
221
+ new_lines = lines[start:]
222
+ events_created, last_uuid = import_lines(
223
+ session, thread_id, new_lines, parser, builder, source=SOURCE, source_id=source_id,
224
+ )
225
+
226
+ # A freshly-created thread that imported nothing (no importable content) is
227
+ # cleaned up so we don't leave an empty thread behind — row AND staged truth
228
+ # record, so no ghost threads/<id>.jsonl survives the commit.
229
+ if is_new_thread and events_created == 0:
230
+ discard_new_thread(session, thread_id)
231
+ thread_id = 0
232
+ is_new_thread = False
233
+ elif thread_id and events_created:
234
+ set_thread_models_from_events(session, thread_id)
235
+
236
+ upsert_import_state(
237
+ session,
238
+ source=SOURCE,
239
+ source_id=source_id,
240
+ thread_id=thread_id or None,
241
+ last_line_count=total,
242
+ last_file_size=0,
243
+ last_message_uuid=last_uuid,
244
+ )
245
+ return ClaudeScienceImportResult(events_created, thread_id or 0, is_new_thread)
246
+
247
+
248
+ def import_claude_science_frame(
249
+ org_uuid: str,
250
+ frame: dict,
251
+ message_rows: list,
252
+ parser: Optional[ClaudeCodeParser] = None,
253
+ builder: Optional[DefaultEventBuilder] = None,
254
+ *,
255
+ session=None,
256
+ ) -> ClaudeScienceImportResult:
257
+ """Import one frame's transcript (atomic per-frame transaction when no session)."""
258
+ parser = parser or ClaudeCodeParser()
259
+ builder = builder or DefaultEventBuilder()
260
+ if session is not None:
261
+ return _run_frame(session, org_uuid, frame, message_rows, parser, builder)
262
+ with get_session() as s:
263
+ result = _run_frame(s, org_uuid, frame, message_rows, parser, builder)
264
+ s.commit()
265
+ return result
266
+
267
+
268
+ # The frame columns the importer reads (title/metadata/timestamps + identity).
269
+ _FRAME_COLUMNS = (
270
+ "id", "parent_frame_id", "root_frame_id", "agent_name", "conversation_type",
271
+ "name", "task_summary", "model", "project_id", "created_at",
272
+ )
273
+
274
+
275
+ def import_claude_science_db(db_path, org_uuid: str) -> ClaudeScienceDbScanResult:
276
+ """Open a Claude Science ``operon-cli.db`` (read-only) and import every frame in
277
+ it under ``org_uuid``. The DB is live, so we open it ``mode=ro`` with a busy
278
+ timeout and never write to it."""
279
+ db_path = Path(db_path)
280
+ summary = ClaudeScienceDbScanResult(0, 0, 0)
281
+ parser = ClaudeCodeParser()
282
+ builder = DefaultEventBuilder()
283
+
284
+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
285
+ try:
286
+ conn.execute("PRAGMA busy_timeout=5000")
287
+ if not conn.execute(
288
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='frames'"
289
+ ).fetchone():
290
+ return summary
291
+
292
+ type_ph = ", ".join("?" for _ in _NON_CONVERSATION_TYPES)
293
+ proj_ph = ", ".join("?" for _ in _SEEDED_PROJECT_IDS)
294
+ frames = conn.execute(
295
+ f"SELECT {', '.join(_FRAME_COLUMNS)} FROM frames "
296
+ f"WHERE conversation_type NOT IN ({type_ph}) "
297
+ f" AND (project_id IS NULL OR project_id NOT IN ({proj_ph})) "
298
+ f"ORDER BY created_at",
299
+ (*_NON_CONVERSATION_TYPES, *_SEEDED_PROJECT_IDS),
300
+ ).fetchall()
301
+
302
+ for frame_row in frames:
303
+ frame = dict(zip(_FRAME_COLUMNS, frame_row))
304
+ frame_id = frame["id"]
305
+ message_rows = conn.execute(
306
+ "SELECT idx, msg_json FROM frame_messages WHERE frame_id = ? ORDER BY idx",
307
+ (frame_id,),
308
+ ).fetchall()
309
+ if not message_rows:
310
+ continue
311
+ summary.frames_processed += 1
312
+ try:
313
+ result = import_claude_science_frame(
314
+ org_uuid, frame, message_rows, parser, builder
315
+ )
316
+ if result.events_created > 0:
317
+ summary.frames_imported += 1
318
+ summary.events_created += result.events_created
319
+ except Exception as e: # noqa: BLE001 — one bad frame must not stop the scan
320
+ # Counted out to the watcher, not just logged — see the same guard in
321
+ # opencode: a silently-skipped frame reads as "nothing new" and leaves
322
+ # `archive status` green while a conversation is missing.
323
+ logger.exception(
324
+ "import_claude_science_db: frame %s failed; skipping", frame_id[:8]
325
+ )
326
+ summary.frames_failed += 1
327
+ summary.errors.append(f"frame {frame_id[:8]}: {e}")
328
+ finally:
329
+ conn.close()
330
+ return summary
@@ -0,0 +1,20 @@
1
+ """cloth session import — cloth is a Claude-Code-shaped terminal harness.
2
+
3
+ cloth writes the same per-session JSONL shape as Claude Code (``user`` / ``assistant``
4
+ lines, plus a ``cloth_meta`` line the parser harmlessly ignores), so its importer
5
+ **delegates to the claude_code line-stream import** under ``source="cloth"`` rather
6
+ than duplicating a parser. Threads land with ``source="cloth"`` and ``source_id`` set
7
+ to the bare session-file stem (the CLI's session uuid — already globally unique — so
8
+ no prefix; see ``watcher.sources.cloth_watcher``). Idempotence, the truth-log seam, and
9
+ incremental watermarking all come from the shared claude_code path unchanged.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from ._result import IncrementalImportResult
15
+ from .claude_code import import_session_incremental
16
+
17
+
18
+ def import_cloth_session_incremental(session_path, source_id: str, *, session=None) -> IncrementalImportResult:
19
+ """Import one cloth CLI transcript into the event log, under ``source="cloth"``."""
20
+ return import_session_incremental(session_path, source_id, source="cloth", session=session)