thread-archive 0.0.2__py3-none-any.whl

thread_archive/_store/schema.py

@@ -0,0 +1,84 @@
+"""Schema provisioning for the SQLite store.
+
+``init_db`` is the schema baseline: ``Base.metadata.create_all`` over the active
+(or given) engine, then ``_add_missing_columns``. Idempotent — existing tables are
+left as-is. There is no alembic; the JSONL truth log + ``reindex`` are the recovery
+primitive.
+
+``create_all`` only issues ``CREATE TABLE`` for tables that don't exist yet, so it
+never adds anything to a table a live index already has. Columns are the one gap that
+can't wait for a reindex: the ORM selects every mapped column, so an index missing a
+newly-added one fails *every* query against that table. Those are ALTERed in on open.
+
+Missing *indexes* are left alone here — they're a correctness/performance property of
+the data, not something a query needs to parse, and building one (e.g. the unique
+dedup index over a multi-million-row events table) is far too much work to do silently
+inside an open. ``verify`` reports an under-enforced schema and ``reindex`` heals it;
+that stays the operator-visible path.
+
+The FTS5 / vector *virtual* tables are not created here — they are built on
+demand by the search layer's ``ensure_index``, since they are derived
+projections, not base tables.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import OperationalError
+
+from . import models  # noqa: F401  (import for side effect: registers tables on Base.metadata)
+from ._base import Base, get_engine
+
+logger = logging.getLogger(__name__)
+
+# table → {column: SQLite type declaration}. Columns added to a table after its
+# baseline shipped; ALTERed in on open when a live index predates them.
+_ADDED_COLUMNS: dict[str, dict[str, str]] = {
+    "import_state": {"last_content_hash": "TEXT"},
+}
+
+
+def _add_missing_columns(engine: Engine) -> None:
+    with engine.begin() as conn:
+        for table, columns in _ADDED_COLUMNS.items():
+            present = conn.execute(
+                text("SELECT name FROM sqlite_master WHERE type='table' AND name=:t"),
+                {"t": table},
+            ).first()
+            if not present:
+                continue
+            have = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
+            for column, decl in columns.items():
+                if column in have:
+                    continue
+                conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {decl}"))
+                logger.info("schema: added %s.%s (%s)", table, column, decl)
+
+
+# Concurrent openers race on a virgin store: ``create_all``'s existence check is
+# not atomic with its CREATEs, so two threads (e.g. the MCP server's warm-models
+# search vs. the first tool call) can both see "no tables" and collide. The lock
+# serializes openers in this process; the retry absorbs a racer in another one.
+_init_lock = threading.Lock()
+
+
+def init_db(engine: Engine | None = None) -> None:
+    """Create all base tables on ``engine`` (or the active engine), then ALTER in any
+    column a pre-existing index predates. Idempotent and safe under concurrent
+    first-open: a lost CREATE race is retried, not raised."""
+    engine = engine or get_engine()
+    with _init_lock:
+        for attempt in (1, 2, 3):
+            try:
+                Base.metadata.create_all(engine)
+                break
+            except OperationalError as exc:
+                if "already exists" not in str(exc) or attempt == 3:
+                    raise
+                # Another process created it between check and CREATE; re-run —
+                # create_all skips what now exists and creates the remainder.
+        _add_missing_columns(engine)

thread_archive/_thread_import/__init__.py

@@ -0,0 +1,40 @@
+"""
+Thread Import Package
+
+Private to thread_archive (hence the underscore): a vendored, dependency-free
+parsing island. Not part of the package's public API until deliberately exposed.
+
+Parsing library for importing conversations from AI services into Thread:
+- Claude Code: Direct from ~/.claude/projects/*.jsonl
+- Cursor: Direct from state.vscdb
+- ChatGPT/Claude Web: From export ZIPs
+
+Architecture:
+- Parsers: Convert provider-specific formats to NormalizedMessage
+- EventBuilder: Converts NormalizedMessage to Thread events
+- Exporters: Extract chat data from local storage (e.g., Cursor's SQLite)
+"""
+
+from .api import ConversationMeta, ImportSource
+from .event_builder import DefaultEventBuilder, EventBuilder, ThreadEvent
+from .exporters import CursorExporter
+from .parsers import ChatGPTParser, ClaudeCodeParser, ClaudeParser, ProviderParser, get_parser
+
+__all__ = [
+    # API
+    "ImportSource",
+    "ConversationMeta",
+    # Event Building
+    "EventBuilder",
+    "DefaultEventBuilder",
+    "ThreadEvent",
+    # Parsers
+    "ChatGPTParser",
+    "ClaudeParser",
+    "ClaudeCodeParser",
+    "ProviderParser",
+    "get_parser",
+    # Exporters
+    "CursorExporter",
+]
+

thread_archive/_thread_import/api.py

@@ -0,0 +1,78 @@
+"""Public API for thread_archive._thread_import.
+
+This module defines the interfaces that consumers (like backend/importer.py)
+should use to access archive data. The key principle is that consumers never
+touch the archive database directly - they go through ImportSource.
+
+This creates a clean boundary:
+- Archive internals can change without breaking consumers
+- Consumers can be tested with mock ImportSource implementations
+- Coalescing and normalization happen in one place (the source)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Iterator, Optional, Protocol
+
+from .parsers.base import NormalizedMessage
+
+
+@dataclass
+class ConversationMeta:
+    """Metadata about a conversation for import decisions.
+
+    This is what ImportSource.list_conversations() yields.
+    Contains enough info to decide whether to import without
+    loading all messages.
+    """
+    source_provider: str
+    provider_conversation_id: str
+    title: Optional[str] = None
+    message_count: int = 0
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+
+class ImportSource(Protocol):
+    """Protocol for anything that can produce messages for import.
+
+    Implementations:
+    - DirectParseSource: Parses export files directly
+    - InMemorySource: For testing
+    """
+
+    def list_conversations(
+        self,
+        provider_filter: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> Iterator[ConversationMeta]:
+        """List available conversations.
+
+        Args:
+            provider_filter: Only include this provider (e.g., "claude-code")
+            limit: Maximum number of conversations to return
+
+        Yields:
+            ConversationMeta for each conversation
+        """
+        ...
+
+    def get_messages(
+        self,
+        provider: str,
+        conversation_id: str,
+        coalesce: bool = True,
+    ) -> Iterator[NormalizedMessage]:
+        """Get messages for a conversation.
+
+        Args:
+            provider: Source provider name
+            conversation_id: Provider's conversation ID
+            coalesce: Whether to coalesce consecutive assistant messages
+
+        Yields:
+            NormalizedMessage objects in order, ready for event creation
+        """
+        ...