thread-archive 0.0.2__py3-none-any.whl

thread_archive/_thread_import/parsers/transformers/ide_context.py

@@ -0,0 +1,158 @@
+"""
+IDE context transformer for Claude Code.
+
+Claude Code embeds IDE context (opened files, selections) as XML-like
+tags in user messages. This transformer extracts them as structured
+ide_context content blocks.
+"""
+
+import re
+from typing import Any, Dict, List, Optional, Tuple, cast
+
+from ..base import ContentBlock
+from ..pipeline.interfaces import RawMessage
+
+# Regex patterns for IDE context tags
+_IDE_OPENED_FILE_PATTERN = re.compile(
+    r"<ide_opened_file>(.*?)</ide_opened_file>", re.DOTALL
+)
+_IDE_SELECTION_PATTERN = re.compile(
+    r"<ide_selection>(.*?)</ide_selection>", re.DOTALL
+)
+
+
+class IDEContextTransformer:
+    """Extracts IDE context from user messages.
+
+    Claude Code embeds IDE state in user messages as XML-like tags:
+    - <ide_opened_file>...</ide_opened_file>
+    - <ide_selection>...</ide_selection>
+
+    This transformer:
+    1. Finds IDE context tags in message content
+    2. Extracts them as ide_context content blocks
+    3. Updates the message text to remove the tags
+
+    Note: This operates on text content, so should run after
+    basic content block extraction.
+    """
+
+    def transform(self, messages: List[RawMessage]) -> List[RawMessage]:
+        """Extract IDE context from messages.
+
+        Args:
+            messages: Messages to transform
+
+        Returns:
+            Messages with IDE context extracted
+        """
+        for msg in messages:
+            if msg.role != "user":
+                continue
+
+            self._extract_ide_context(msg)
+
+        return messages
+
+    def _extract_ide_context(self, msg: RawMessage) -> None:
+        """Extract IDE context from a user message.
+
+        Modifies msg.content_blocks in place.
+        """
+        # Find text blocks to process
+        text_blocks = [
+            (i, b)
+            for i, b in enumerate(msg.content_blocks)
+            if b.get("type") == "text"
+        ]
+
+        if not text_blocks:
+            # Try raw content if no blocks yet
+            if isinstance(msg.content, str):
+                cleaned, ide_blocks = self._extract_from_text(msg.content)
+                if ide_blocks:
+                    # Update content
+                    msg.content = cleaned
+                    # Add ide_context blocks
+                    base_seq = len(msg.content_blocks)
+                    for i, block in enumerate(ide_blocks):
+                        block["seq"] = base_seq + i
+                        msg.content_blocks.append(cast(ContentBlock, block))
+                    # Update existing text block if present
+                    if msg.content_blocks and msg.content_blocks[0].get("type") == "text":
+                        cast(Dict[str, Any], msg.content_blocks[0])["text"] = cleaned
+            return
+
+        # Process each text block
+        ide_blocks_to_add: List[Dict[str, Any]] = []
+        for idx, block in text_blocks:
+            text = block.get("text", "")
+            if not text:
+                continue
+
+            cleaned, extracted = self._extract_from_text(text)
+
+            # Update the text block
+            cast(Dict[str, Any], block)["text"] = cleaned
+
+            # Collect IDE blocks
+            ide_blocks_to_add.extend(extracted)
+
+        # Add IDE blocks at the end
+        if ide_blocks_to_add:
+            base_seq = len(msg.content_blocks)
+            for i, block in enumerate(ide_blocks_to_add):
+                block["seq"] = base_seq + i
+                msg.content_blocks.append(cast(ContentBlock, block))
+
+    def _extract_from_text(
+        self, text: str
+    ) -> Tuple[str, List[Dict[str, Any]]]:
+        """Extract IDE context tags from text.
+
+        Returns:
+            Tuple of (cleaned_text, list of ide_context blocks)
+        """
+        ide_blocks: List[Dict[str, Any]] = []
+
+        # Extract <ide_opened_file> tags
+        for match in _IDE_OPENED_FILE_PATTERN.finditer(text):
+            content = match.group(1).strip()
+            file_path = self._extract_file_path(content)
+
+            ide_blocks.append({
+                "type": "ide_context",
+                "context_type": "opened_file",
+                "file_path": file_path,
+                "raw_content": content,
+                "seq": 0,  # Will be renumbered
+            })
+
+        # Extract <ide_selection> tags
+        for match in _IDE_SELECTION_PATTERN.finditer(text):
+            content = match.group(1).strip()
+
+            ide_blocks.append({
+                "type": "ide_context",
+                "context_type": "selection",
+                "raw_content": content,
+                "seq": 0,  # Will be renumbered
+            })
+
+        # Remove tags from text
+        cleaned = _IDE_OPENED_FILE_PATTERN.sub("", text)
+        cleaned = _IDE_SELECTION_PATTERN.sub("", cleaned)
+        cleaned = cleaned.strip()
+
+        return cleaned, ide_blocks
+
+    def _extract_file_path(self, content: str) -> Optional[str]:
+        """Extract file path from opened file content.
+
+        Content is typically: "The user opened the file {path} in the IDE..."
+        """
+        if "opened the file " in content:
+            path_match = re.search(r"opened the file ([^\s]+)", content)
+            if path_match:
+                return path_match.group(1)
+        return None

thread_archive/_thread_import/parsers/transformers/thinking_merge.py

@@ -0,0 +1,68 @@
+"""
+Thinking merge transformer for Claude Code.
+
+In Claude Code exports, thinking-only messages (messages with only thinking
+blocks and no text) should be merged into their parent assistant message's
+content_blocks.
+"""
+
+from typing import Dict, List, Set
+
+from ..pipeline.interfaces import RawMessage
+
+
+class ThinkingMergeTransformer:
+    """Merges thinking-only messages into parent assistant messages.
+
+    In the normalized data model, thinking blocks should be nested within
+    the parent message's content_blocks, not as separate messages.
+
+    This transformer:
+    1. Identifies messages with only thinking blocks (no text)
+    2. Appends their thinking blocks to the parent's content_blocks
+    3. Removes the thinking-only messages from output
+
+    Original provider data is preserved in provider_data.
+    """
+
+    def transform(self, messages: List[RawMessage]) -> List[RawMessage]:
+        """Merge thinking-only messages into parents.
+
+        Args:
+            messages: Messages to transform
+
+        Returns:
+            Messages with thinking blocks merged
+        """
+        if not messages:
+            return messages
+
+        # Build lookup by ID
+        msg_by_id: Dict[str, RawMessage] = {
+            m.provider_message_id: m for m in messages
+        }
+
+        to_remove: Set[str] = set()
+
+        for msg in messages:
+            blocks = msg.content_blocks
+            has_text = any(
+                b.get("type") == "text" and b.get("text") for b in blocks
+            )
+            has_thinking = any(b.get("type") == "thinking" for b in blocks)
+
+            # If message has thinking but no text, merge into parent
+            if has_thinking and not has_text:
+                parent_id = msg.provider_parent_id
+                if parent_id and parent_id in msg_by_id:
+                    parent = msg_by_id[parent_id]
+                    # Only merge into assistant messages
+                    if parent.role == "assistant":
+                        # Append thinking blocks to parent
+                        for block in blocks:
+                            if block.get("type") == "thinking":
+                                parent.content_blocks.append(block)
+                        to_remove.add(msg.provider_message_id)
+
+        # Remove merged messages
+        return [m for m in messages if m.provider_message_id not in to_remove]

thread_archive/_thread_import/parsers/types/__init__.py

@@ -0,0 +1,56 @@
+"""
+Type definitions for provider-specific export formats.
+
+Each module defines TypedDict schemas for the raw input format from that provider.
+These are used for type checking and IDE support when parsing exports.
+"""
+
+from .chatgpt import (
+    ChatGPTAuthor,
+    ChatGPTContent,
+    ChatGPTConversation,
+    ChatGPTExport,
+    ChatGPTMessage,
+    ChatGPTNode,
+)
+from .claude import (
+    ClaudeContentBlock,
+    ClaudeConversation,
+    ClaudeExport,
+    ClaudeMessage,
+)
+from .claude_code import (
+    ClaudeCodeExport,
+    ClaudeCodeLine,
+    ClaudeCodeMessage,
+    ClaudeCodeSession,
+)
+from .cursor import (
+    CursorConversation,
+    CursorExport,
+    CursorMessage,
+)
+
+__all__ = [
+    # ChatGPT
+    "ChatGPTAuthor",
+    "ChatGPTContent",
+    "ChatGPTMessage",
+    "ChatGPTNode",
+    "ChatGPTConversation",
+    "ChatGPTExport",
+    # Claude
+    "ClaudeContentBlock",
+    "ClaudeMessage",
+    "ClaudeConversation",
+    "ClaudeExport",
+    # Claude Code
+    "ClaudeCodeMessage",
+    "ClaudeCodeLine",
+    "ClaudeCodeSession",
+    "ClaudeCodeExport",
+    # Cursor
+    "CursorMessage",
+    "CursorConversation",
+    "CursorExport",
+]

thread_archive/_thread_import/parsers/types/chatgpt.py

@@ -0,0 +1,99 @@
+"""
+TypedDict schemas for ChatGPT export format.
+
+Export source: ChatGPT web UI -> Settings -> Data controls -> Export data
+File structure: ZIP archive containing conversations.json
+
+The conversations.json contains an array of conversation objects with:
+- mapping: Dict of message_id -> message node (tree structure)
+- current_node: ID of the leaf node in the "active" conversation path
+- title, create_time, update_time: Conversation metadata
+"""
+
+from typing import Any, Dict, List, Optional, TypedDict
+
+
+class ChatGPTAuthor(TypedDict, total=False):
+    """Author information for a ChatGPT message."""
+    role: str  # "user", "assistant", "system", "tool"
+    name: Optional[str]  # Tool/plugin name when role="tool"
+    metadata: Dict[str, Any]
+
+
+class ChatGPTContent(TypedDict, total=False):
+    """Content structure for a ChatGPT message.
+
+    ChatGPT uses various content_type values:
+    - "text": Regular text content
+    - "thoughts", "analysis", "reasoning_recap": Thinking blocks (o1 models)
+    - "user_editable_context", "model_editable_context": Custom instructions
+    - "code", "commentary", "metadata", "system": Structural content
+    """
+    content_type: str
+    parts: List[Any]  # Array of content parts (strings, dicts)
+    text: Optional[str]  # Alternative to parts array
+
+
+class ChatGPTMessageMetadata(TypedDict, total=False):
+    """Metadata for a ChatGPT message."""
+    model_slug: Optional[str]
+    model: Optional[str]
+    is_visually_hidden_from_conversation: Optional[bool]
+    is_user_system_message: Optional[bool]
+    parent_id: Optional[str]
+    content_type: Optional[str]
+    tool_calls: Optional[List[Dict[str, Any]]]
+    tool_call: Optional[Dict[str, Any]]
+
+
+class ChatGPTMessage(TypedDict, total=False):
+    """A single message in a ChatGPT conversation."""
+    id: str
+    author: ChatGPTAuthor
+    content: ChatGPTContent
+    create_time: Optional[float]  # Unix timestamp
+    update_time: Optional[float]  # Unix timestamp
+    metadata: ChatGPTMessageMetadata
+    status: Optional[str]
+    end_turn: Optional[bool]
+    weight: Optional[float]
+    recipient: Optional[str]
+
+
+class ChatGPTNode(TypedDict, total=False):
+    """A node in the ChatGPT message tree (from mapping field)."""
+    id: str
+    parent: Optional[str]  # Parent node ID
+    children: List[str]  # Child node IDs
+    message: Optional[ChatGPTMessage]  # None for stub nodes
+
+
+class ChatGPTConversation(TypedDict, total=False):
+    """A single ChatGPT conversation."""
+    id: str
+    title: str
+    create_time: float
+    update_time: float
+    mapping: Dict[str, ChatGPTNode]  # message_id -> node
+    current_node: Optional[str]  # Leaf of active path
+    moderation_results: Optional[List[Any]]
+    plugin_ids: Optional[List[str]]
+    gizmo_id: Optional[str]
+    conversation_template_id: Optional[str]
+    safe_urls: Optional[List[str]]
+    is_archived: Optional[bool]
+
+
+class ChatGPTUser(TypedDict, total=False):
+    """User information from ChatGPT export."""
+    id: str
+    name: str
+    email: str
+
+
+class ChatGPTExport(TypedDict, total=False):
+    """Complete ChatGPT data export bundle."""
+    conversations: List[ChatGPTConversation]
+    user: Optional[ChatGPTUser]
+    message_feedback: Optional[Dict[str, Any]]
+    model_comparisons: Optional[List[Any]]

thread_archive/_thread_import/parsers/types/claude.py

@@ -0,0 +1,120 @@
+"""
+TypedDict schemas for Claude.ai export format.
+
+Export source: Claude.ai -> Settings -> Account -> Export Data
+File structure: ZIP archive containing:
+- conversations.json - Chat conversations with messages (primary)
+- memories.json - Claude's memory about the user (optional)
+- projects.json - Claude Projects and attached docs (optional)
+- users.json - Account information (optional)
+
+Key differences from other providers:
+- No thinking blocks (server-side only, not included in exports)
+- No branching (strictly linear conversations)
+- Uses "human" role where others use "user"
+"""
+
+from typing import Any, Dict, List, Optional, TypedDict
+
+
+class ClaudeContentBlock(TypedDict, total=False):
+    """A content block in a Claude message.
+
+    Block types:
+    - text: Plain text content
+    - tool_use: Tool/function call
+    - tool_result: Tool/function result
+    - thinking: Extended thinking (NOT present in web exports)
+    - token_budget: Internal token tracking
+    """
+    type: str  # "text", "tool_use", "tool_result", "thinking", "token_budget"
+    text: Optional[str]  # For text blocks
+    thinking: Optional[str]  # For thinking blocks
+    signature: Optional[str]  # For thinking blocks
+    name: Optional[str]  # For tool_use and tool_result
+    input: Optional[Dict[str, Any]]  # For tool_use
+    content: Optional[Any]  # For tool_result
+    is_error: Optional[bool]  # For tool_result
+    display_content: Optional[Any]
+    start_timestamp: Optional[str]
+    stop_timestamp: Optional[str]
+
+
+class ClaudeAttachment(TypedDict, total=False):
+    """File attachment in a Claude message."""
+    id: str
+    file_name: str
+    file_type: str
+    file_size: Optional[int]
+    extracted_content: Optional[str]
+
+
+class ClaudeMessage(TypedDict, total=False):
+    """A single message in a Claude conversation."""
+    uuid: str
+    text: str  # Legacy field, may exist alongside content
+    content: List[ClaudeContentBlock]  # Structured content blocks
+    sender: str  # "human" or "assistant"
+    created_at: str  # ISO timestamp
+    updated_at: Optional[str]  # ISO timestamp
+    attachments: List[ClaudeAttachment]
+    files: List[Dict[str, Any]]
+
+
+class ClaudeAccount(TypedDict, total=False):
+    """Account reference in a conversation."""
+    uuid: str
+
+
+class ClaudeProject(TypedDict, total=False):
+    """Project reference in a conversation."""
+    uuid: str
+
+
+class ClaudeConversation(TypedDict, total=False):
+    """A single Claude conversation."""
+    uuid: str
+    name: str
+    summary: Optional[str]
+    created_at: str
+    updated_at: str
+    account: ClaudeAccount
+    project: Optional[ClaudeProject]
+    chat_messages: List[ClaudeMessage]
+
+
+class ClaudeMemory(TypedDict, total=False):
+    """User memory from Claude export."""
+    content: str
+    created_at: str
+    updated_at: str
+
+
+class ClaudeMemories(TypedDict, total=False):
+    """Memories section of Claude export."""
+    user_memories: List[ClaudeMemory]
+    project_memories: Dict[str, str]  # project_uuid -> memory text
+
+
+class ClaudeProjectFull(TypedDict, total=False):
+    """Full project information from Claude export."""
+    uuid: str
+    name: str
+    description: Optional[str]
+    created_at: str
+    updated_at: str
+
+
+class ClaudeUser(TypedDict, total=False):
+    """User information from Claude export."""
+    uuid: str
+    full_name: str
+    email: Optional[str]
+
+
+class ClaudeExport(TypedDict, total=False):
+    """Complete Claude data export bundle."""
+    conversations: List[ClaudeConversation]
+    memories: Optional[ClaudeMemories]
+    projects: Optional[List[ClaudeProjectFull]]
+    users: Optional[List[ClaudeUser]]

thread_archive/_thread_import/parsers/types/claude_code.py

@@ -0,0 +1,139 @@
+"""
+TypedDict schemas for Claude Code CLI session format.
+
+Storage location: ~/.claude/projects/<project-path>/<session-id>.jsonl
+Where <project-path> is the working directory with / replaced by -
+Example: /home/user/project -> -home-user-project
+
+Sessions are JSONL (one JSON object per line) with message types:
+- type: "user" - User messages with message.content
+- type: "assistant" - Assistant responses with content blocks
+- type: "system" - System events (commands, file snapshots, compaction)
+- type: "summary" - Context continuation summaries
+- type: "file-history-snapshot" - File state snapshots
+
+Key characteristics:
+- Model-specific thinking (Opus has extended thinking, Sonnet/Haiku don't)
+- Context compaction via compact_boundary system messages
+- IDE context embedded as XML tags in user messages
+- Tool results in user messages (not coalesced like ChatGPT)
+"""
+
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+
+
+class ClaudeCodeContentBlock(TypedDict, total=False):
+    """A content block in a Claude Code message.
+
+    Block types mirror Claude's API:
+    - text: Plain text content
+    - thinking: Extended thinking (signature-verified)
+    - tool_use: Tool call
+    - tool_result: Tool execution result
+    """
+    type: str  # "text", "thinking", "tool_use", "tool_result"
+    text: Optional[str]  # For text blocks
+    thinking: Optional[str]  # For thinking blocks
+    signature: Optional[str]  # For thinking blocks
+    id: Optional[str]  # For tool_use
+    name: Optional[str]  # For tool_use
+    input: Optional[Dict[str, Any]]  # For tool_use
+    tool_use_id: Optional[str]  # For tool_result
+    content: Optional[Any]  # For tool_result
+
+
+class ClaudeCodeUsage(TypedDict, total=False):
+    """Token usage information."""
+    input_tokens: int
+    output_tokens: int
+    cache_creation_input_tokens: Optional[int]
+    cache_read_input_tokens: Optional[int]
+
+
+class ClaudeCodeMessage(TypedDict, total=False):
+    """The message object within a Claude Code line.
+
+    For user messages: content is a string or List[ContentBlock]
+    For assistant messages: content is List[ContentBlock]
+    """
+    id: Optional[str]
+    role: str  # "user" or "assistant"
+    content: Union[str, List[ClaudeCodeContentBlock]]
+    model: Optional[str]
+    stop_reason: Optional[str]
+    usage: Optional[ClaudeCodeUsage]
+
+
+class ClaudeCodeCompactMetadata(TypedDict, total=False):
+    """Metadata for context compaction events."""
+    preTokens: int
+    postTokens: int
+    summarizedMessages: List[str]
+
+
+class ClaudeCodeLine(TypedDict, total=False):
+    """A single line/record in a Claude Code JSONL session file.
+
+    type determines the structure:
+    - "user": User message with message.content
+    - "assistant": Assistant response with message.content blocks
+    - "system": System event (command, compaction boundary)
+    - "summary": Context continuation summary
+    - "file-history-snapshot": File state snapshot
+    """
+    type: str  # "user", "assistant", "system", "summary", "file-history-snapshot"
+    uuid: str
+    parentUuid: Optional[str]
+    sessionId: str
+    timestamp: str  # ISO 8601
+    message: Optional[ClaudeCodeMessage]
+
+    # System message fields
+    subtype: Optional[str]  # "compact_boundary", etc.
+    content: Optional[str]  # For system messages
+    level: Optional[str]  # Log level
+
+    # Context compaction fields
+    logicalParentUuid: Optional[str]  # Points to pre-compaction message
+    compactMetadata: Optional[ClaudeCodeCompactMetadata]
+
+    # Summary fields
+    summary: Optional[str]
+    summarizes: Optional[List[str]]
+    summarizedMessages: Optional[List[str]]
+
+    # File snapshot fields
+    files: Optional[List[Dict[str, Any]]]
+    snapshot: Optional[Dict[str, Any]]
+
+    # Session metadata
+    cwd: Optional[str]  # Working directory
+    gitBranch: Optional[str]
+    version: Optional[str]  # Claude Code version
+    requestId: Optional[str]
+
+    # Branching
+    isSidechain: Optional[bool]
+
+    # Thinking metadata
+    thinkingMetadata: Optional[Dict[str, Any]]
+
+    # Todo tracking
+    todos: Optional[List[Dict[str, Any]]]
+
+
+class ClaudeCodeSession(TypedDict, total=False):
+    """A parsed Claude Code session."""
+    session_id: str
+    path: Optional[str]
+    lines: List[ClaudeCodeLine]
+    parse_errors: Optional[List[Dict[str, Any]]]
+    is_subagent: Optional[bool]
+    parent_session_id: Optional[str]
+
+
+class ClaudeCodeExport(TypedDict, total=False):
+    """Claude Code export bundle (multiple sessions)."""
+    provider: Literal["claude-code"]
+    sessions: List[ClaudeCodeSession]
+    export_metadata: Optional[Dict[str, Any]]