thread-archive 0.0.2__py3-none-any.whl

thread_archive/_thread_import/parsers/pipeline/interfaces.py

@@ -0,0 +1,191 @@
+"""
+Protocol definitions for pipeline components.
+
+These Protocols define the interfaces for pluggable pipeline components.
+Each component can be implemented independently and composed together.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Protocol, TypeVar
+
+# Import from parent to avoid circular imports
+# These will be available once the module structure is in place
+from ..base import ContentBlock, NormalizedMessage
+
+# Type variable for generic input type. Contravariant: the Parser Protocol only
+# consumes it (parse(data: T_Input)), never produces it.
+T_Input = TypeVar("T_Input", contravariant=True)
+
+
+@dataclass
+class RawMessage:
+    """Intermediate representation between parse and normalize.
+
+    This is the provider-specific parsed data before normalization.
+    It preserves all provider-specific fields while providing a
+    common structure for transformers to operate on.
+
+    Attributes:
+        provider_message_id: Original message ID from provider
+        provider_conversation_id: Original conversation ID
+        provider_parent_id: Parent message ID (for tree structures)
+        role: Raw role value (not yet normalized)
+        content: Raw content (provider-specific structure)
+        content_type: Content type indicator (if applicable)
+        timestamp: Raw timestamp (provider-specific format)
+        is_active_path: Whether on main conversation thread
+        is_stub: Whether this is a placeholder node
+        metadata: Additional provider-specific metadata
+        raw_data: Complete original data for reference
+        children: IDs of child messages (for tree traversal)
+    """
+
+    provider_message_id: str
+    provider_conversation_id: str
+    provider_parent_id: Optional[str] = None
+    role: Optional[str] = None
+    content: Any = None
+    content_type: Optional[str] = None
+    timestamp: Any = None  # Provider-specific format
+    timestamp_iso: Optional[str] = None  # Normalized ISO format
+    message_order: Optional[int] = None
+    is_active_path: bool = True
+    is_stub: bool = False
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+    children: List[str] = field(default_factory=list)
+
+    # Content blocks (built during parsing or transformation)
+    content_blocks: List[ContentBlock] = field(default_factory=list)
+
+    # Conversation-level metadata
+    conversation_title: Optional[str] = None
+    conversation_metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __hash__(self) -> int:
+        """Allow use in sets (based on message ID)."""
+        return hash(self.provider_message_id)
+
+    def __eq__(self, other: object) -> bool:
+        """Equality based on message ID."""
+        if not isinstance(other, RawMessage):
+            return False
+        return self.provider_message_id == other.provider_message_id
+
+
+class Parser(Protocol[T_Input]):
+    """Protocol for parsing raw provider data.
+
+    Parsers convert provider-specific export data into RawMessage objects.
+    This is the first phase of the pipeline.
+
+    Type Parameters:
+        T_Input: The type of raw input data (e.g., ChatGPTExport)
+    """
+
+    def parse(self, data: T_Input) -> List[RawMessage]:
+        """Parse raw provider data into intermediate representation.
+
+        Args:
+            data: Raw provider export data
+
+        Returns:
+            List of RawMessage objects
+        """
+        ...
+
+
+class Transformer(Protocol):
+    """Protocol for transforming messages.
+
+    Transformers modify the list of RawMessages. Common uses:
+    - Coalescing tool calls into parent messages (ChatGPT)
+    - Merging thinking blocks into parent (Claude Code)
+    - Computing active path (ChatGPT)
+    - Extracting IDE context (Claude Code)
+
+    Transformers are applied in sequence, so order matters.
+    """
+
+    def transform(self, messages: List[RawMessage]) -> List[RawMessage]:
+        """Transform a list of messages.
+
+        Args:
+            messages: Input messages
+
+        Returns:
+            Transformed messages (may be modified in place or new list)
+        """
+        ...
+
+
+class Normalizer(Protocol):
+    """Protocol for normalizing messages to canonical format.
+
+    Normalizers convert RawMessage objects into NormalizedMessage
+    dictionaries that match the unified schema.
+    """
+
+    def normalize(self, raw: RawMessage) -> NormalizedMessage:
+        """Convert a RawMessage to NormalizedMessage.
+
+        Args:
+            raw: Intermediate message representation
+
+        Returns:
+            Canonical NormalizedMessage dictionary
+        """
+        ...
+
+
+@dataclass
+class ValidationContext:
+    """Context for validation across a conversation.
+
+    A mutable interface that validators update as they process messages.
+    """
+
+    conversation_id: str
+    source_provider: str
+    message_count: int = 0
+    first_user_msg_seen: bool = False
+    first_user_msg_has_context: bool = False
+    assistant_msg_count: int = 0
+    assistant_msgs_with_thinking: int = 0
+    assistant_msgs_with_tools: int = 0
+    thinking_required_msgs: int = 0
+    thinking_required_msgs_with_thinking: int = 0
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+
+    def add_error(self, msg: str) -> None:
+        """Add a validation error."""
+        self.errors.append(msg)
+
+    def add_warning(self, msg: str) -> None:
+        """Add a validation warning."""
+        self.warnings.append(msg)
+
+    @property
+    def has_errors(self) -> bool:
+        """Check if any errors were recorded."""
+        return len(self.errors) > 0
+
+
+class Validator(Protocol):
+    """Protocol for validating messages.
+
+    Validators check messages against rules and update the
+    ValidationContext with errors/warnings.
+    """
+
+    def validate(
+        self, messages: List[NormalizedMessage], context: ValidationContext
+    ) -> None:
+        """Validate messages and update context.
+
+        Args:
+            messages: Messages to validate
+            context: Validation context to update with results
+        """
+        ...

thread_archive/_thread_import/parsers/transformers/__init__.py

@@ -0,0 +1,22 @@
+"""
+Transformers for the parser pipeline.
+
+Transformers modify RawMessage lists between parse and normalize phases.
+Common uses:
+- Coalescing tool messages into parents (ChatGPT)
+- Merging thinking blocks (Claude Code)
+- Computing active paths (ChatGPT)
+- Extracting IDE context (Claude Code)
+"""
+
+from .active_path import ActivePathTransformer
+from .coalescing import ToolCoalescingTransformer
+from .ide_context import IDEContextTransformer
+from .thinking_merge import ThinkingMergeTransformer
+
+__all__ = [
+    "ToolCoalescingTransformer",
+    "ActivePathTransformer",
+    "ThinkingMergeTransformer",
+    "IDEContextTransformer",
+]

thread_archive/_thread_import/parsers/transformers/active_path.py

@@ -0,0 +1,87 @@
+"""
+Active path transformer for ChatGPT.
+
+ChatGPT conversations are tree-structured. The "active path" is the
+main thread from root to current_node. This transformer marks messages
+on or off the active path.
+"""
+
+from typing import Dict, List, Optional, Set
+
+from ..pipeline.interfaces import RawMessage
+
+
+class ActivePathTransformer:
+    """Computes and marks the active conversation path.
+
+    ChatGPT's current_node field identifies the leaf of the "active"
+    conversation thread. This transformer:
+    1. Walks from current_node to root to find the active path
+    2. Marks each message with is_active_path flag
+
+    This is run before coalescing so only active-path messages
+    get coalesced into their parents.
+    """
+
+    def __init__(self, current_node: Optional[str] = None):
+        """Initialize with optional current_node.
+
+        Args:
+            current_node: Leaf node ID of active path. If not provided,
+                          tries to extract from messages' conversation metadata.
+        """
+        self.current_node = current_node
+
+    def transform(self, messages: List[RawMessage]) -> List[RawMessage]:
+        """Mark messages with active path status.
+
+        Args:
+            messages: Messages to transform
+
+        Returns:
+            Same messages with is_active_path updated
+        """
+        if not messages:
+            return messages
+
+        # Try to get current_node from conversation metadata
+        current_node = self.current_node
+        if not current_node:
+            for msg in messages:
+                conv_meta = msg.conversation_metadata
+                if conv_meta and "current_node" in conv_meta:
+                    current_node = conv_meta["current_node"]
+                    break
+
+        if not current_node:
+            # No current_node - assume all messages are active
+            return messages
+
+        # Build parent lookup
+        parent_map: Dict[str, Optional[str]] = {}
+        for msg in messages:
+            parent_map[msg.provider_message_id] = msg.provider_parent_id
+
+        # Walk from current_node to root
+        active_ids = self._build_active_path(current_node, parent_map)
+
+        # Mark messages
+        for msg in messages:
+            msg.is_active_path = msg.provider_message_id in active_ids
+
+        return messages
+
+    def _build_active_path(
+        self, current_node: str, parent_map: Dict[str, Optional[str]]
+    ) -> Set[str]:
+        """Walk from current_node to root to build active path set."""
+        active_ids: Set[str] = set()
+        node_id: Optional[str] = current_node
+        hop_guard = 0
+
+        while node_id and hop_guard < 10_000:
+            active_ids.add(node_id)
+            node_id = parent_map.get(node_id)
+            hop_guard += 1
+
+        return active_ids

thread_archive/_thread_import/parsers/transformers/coalescing.py

@@ -0,0 +1,389 @@
+"""
+Tool coalescing transformer for ChatGPT.
+
+ChatGPT exports tool calls, thinking, and results as separate messages
+linked via parent_id. This transformer coalesces them into content_blocks
+on the parent assistant message.
+"""
+
+from typing import Any, Dict, List, Set, cast
+
+from ..base import ContentBlock
+from ..pipeline.interfaces import RawMessage
+
+# Content types that indicate thinking/reasoning
+THINKING_CONTENT_TYPES = {"thoughts", "analysis", "reasoning_recap"}
+
+# Content types that indicate structural/scaffolding messages
+STRUCTURAL_CONTENT_TYPES = {"code", "commentary", "metadata", "system"}
+
+# Content types that represent system context
+CONTEXT_CONTENT_TYPES = {"user_editable_context", "model_editable_context"}
+
+
+class ToolCoalescingTransformer:
+    """Coalesces tool/thinking messages into parent assistant messages.
+
+    ChatGPT's export format stores:
+    - Tool calls as separate assistant messages with tool_calls in metadata
+    - Tool results as separate tool-role messages
+    - Thinking as separate messages with content_type in THINKING_CONTENT_TYPES
+    - System context as separate messages with content_type in CONTEXT_CONTENT_TYPES
+
+    This transformer:
+    1. Identifies "primary" messages (user/assistant with displayable content)
+    2. Finds child messages that should be coalesced (tool, thinking, context)
+    3. Converts coalesced messages to content_blocks on the parent
+    4. Removes coalesced messages from the output list
+
+    Only messages on the active path are coalesced. Branch messages remain separate.
+    """
+
+    def transform(self, messages: List[RawMessage]) -> List[RawMessage]:
+        """Coalesce tool and thinking messages into parents.
+
+        Args:
+            messages: Messages to transform (should have is_active_path set)
+
+        Returns:
+            Transformed messages with coalesced content
+        """
+        if not messages:
+            return messages
+
+        # Build lookup structures
+        msg_by_id: Dict[str, RawMessage] = {
+            m.provider_message_id: m for m in messages
+        }
+        children_by_parent: Dict[str, List[str]] = {}
+        for msg in messages:
+            if msg.provider_parent_id:
+                children_by_parent.setdefault(msg.provider_parent_id, []).append(
+                    msg.provider_message_id
+                )
+
+        # Sort by message_order for consistent processing
+        sorted_messages = sorted(messages, key=lambda m: m.message_order or 0)
+
+        result: List[RawMessage] = []
+        coalesced_ids: Set[str] = set()
+
+        for msg in sorted_messages:
+            if msg.provider_message_id in coalesced_ids:
+                continue
+
+            if self._is_primary_message(msg):
+                # Collect children to coalesce
+                children_to_coalesce = self._collect_coalesce_candidates(
+                    msg.provider_message_id,
+                    children_by_parent,
+                    msg_by_id,
+                )
+
+                # Mark as coalesced
+                for child_id in children_to_coalesce:
+                    coalesced_ids.add(child_id)
+
+                # Build content blocks from primary + children
+                self._build_content_blocks(
+                    msg,
+                    [msg_by_id[cid] for cid in children_to_coalesce if cid in msg_by_id],
+                )
+
+                result.append(msg)
+            else:
+                # Non-primary that wasn't coalesced - keep as separate
+                result.append(msg)
+
+        return result
+
+    def _is_primary_message(self, msg: RawMessage) -> bool:
+        """Determine if message should be a primary row.
+
+        Primary messages:
+        - User messages with displayable content
+        - Assistant messages with text content
+        - System messages
+
+        Non-primary (coalesce candidates):
+        - Tool role messages
+        - Thinking/reasoning messages
+        - Stubs
+        """
+        if msg.is_stub:
+            return False
+
+        role = (msg.role or "").lower()
+        content_type = (msg.content_type or "").lower()
+
+        # Tool messages are coalesced
+        if role == "tool":
+            return False
+
+        # Thinking/reasoning messages are coalesced
+        if content_type in THINKING_CONTENT_TYPES:
+            return False
+
+        # System context is coalesced
+        if content_type in CONTEXT_CONTENT_TYPES:
+            return False
+
+        # Structural content is coalesced
+        if content_type in STRUCTURAL_CONTENT_TYPES:
+            return False
+
+        # Check for actual displayable content
+        content = msg.content
+        if isinstance(content, dict):
+            parts = content.get("parts", [])
+            text = content.get("text", "")
+            if not parts and not text:
+                return False
+
+        return role in ("user", "assistant", "system")
+
+    def _collect_coalesce_candidates(
+        self,
+        parent_id: str,
+        children_by_parent: Dict[str, List[str]],
+        msg_by_id: Dict[str, RawMessage],
+    ) -> List[str]:
+        """Recursively collect child messages to coalesce.
+
+        Only coalesces messages on the active path that are:
+        - Tool messages
+        - Thinking/reasoning messages
+        - Context messages
+        - Structural messages
+        """
+        result: List[str] = []
+        to_process = list(children_by_parent.get(parent_id, []))
+
+        while to_process:
+            child_id = to_process.pop(0)
+            child_msg = msg_by_id.get(child_id)
+            if not child_msg:
+                continue
+
+            # Only coalesce active path messages
+            if not child_msg.is_active_path:
+                continue
+
+            role = (child_msg.role or "").lower()
+            content_type = (child_msg.content_type or "").lower()
+
+            should_coalesce = (
+                role == "tool"
+                or content_type in THINKING_CONTENT_TYPES
+                or content_type in CONTEXT_CONTENT_TYPES
+                or content_type in STRUCTURAL_CONTENT_TYPES
+                or child_msg.is_stub
+            )
+
+            if should_coalesce:
+                result.append(child_id)
+                # Continue to this message's children
+                to_process.extend(children_by_parent.get(child_id, []))
+
+        return result
+
+    def _build_content_blocks(
+        self, primary: RawMessage, coalesced: List[RawMessage]
+    ) -> None:
+        """Build content_blocks from primary and coalesced messages.
+
+        Modifies primary.content_blocks in place.
+        """
+        blocks: List[ContentBlock] = []
+        seq = 0
+
+        # Extract from primary
+        primary_blocks, seq = self._extract_blocks(primary, seq)
+        blocks.extend(primary_blocks)
+
+        # Add from coalesced
+        for child in coalesced:
+            child_blocks, seq = self._extract_blocks(child, seq)
+            blocks.extend(child_blocks)
+
+        primary.content_blocks = blocks
+
+    def _extract_blocks(
+        self, msg: RawMessage, start_seq: int
+    ) -> tuple[List[ContentBlock], int]:
+        """Extract content blocks from a message."""
+        if msg.is_stub:
+            return [], start_seq
+
+        content = msg.content
+        content_type = (msg.content_type or "").lower()
+        role = (msg.role or "").lower()
+        msg_id = msg.provider_message_id
+        metadata = msg.metadata
+
+        # Exclusive cases (tool role / thinking / system context) each fully
+        # handle the message and return early.
+        special = self._extract_special_blocks(
+            content, content_type, role, msg_id, metadata, start_seq
+        )
+        if special is not None:
+            return special
+
+        # Additive sections: metadata tool calls, then regular text, then images.
+        blocks: List[ContentBlock] = []
+        seq = start_seq
+        seq = self._append_metadata_tool_calls(blocks, metadata, msg_id, seq)
+        seq = self._append_text(blocks, content, seq)
+        seq = self._append_images(blocks, content, seq)
+        return blocks, seq
+
+    def _extract_special_blocks(
+        self,
+        content: Any,
+        content_type: str,
+        role: str,
+        msg_id: str,
+        metadata: Dict[str, Any],
+        seq: int,
+    ) -> "tuple[List[ContentBlock], int] | None":
+        """Handle the exclusive content cases that fully consume a message.
+
+        Returns ``(blocks, next_seq)`` when one applies, else ``None`` so the
+        caller continues with the additive (tool_calls/text/image) sections.
+        """
+        # Tool role -> tool_result
+        if role == "tool":
+            tool_name = metadata.get("author_name") or "unknown"
+            return [{
+                "type": "tool_result",
+                "name": tool_name,
+                "content": content,
+                "seq": seq,
+                "provider_message_id": msg_id,
+            }], seq + 1
+
+        # Thinking content
+        if content_type in THINKING_CONTENT_TYPES:
+            text = self._extract_text(content)
+            if text:
+                return [cast(ContentBlock, {
+                    "type": "thinking",
+                    "text": text,
+                    "thinking_type": content_type,
+                    "seq": seq,
+                    "provider_message_id": msg_id,
+                })], seq + 1
+
+        # System context
+        if content_type in CONTEXT_CONTENT_TYPES:
+            text = self._extract_text(content)
+            if text:
+                return [{
+                    "type": "system_context",
+                    "text": text,
+                    "context_type": content_type,
+                    "seq": seq,
+                }], seq + 1
+
+        return None
+
+    def _append_metadata_tool_calls(
+        self,
+        blocks: List[ContentBlock],
+        metadata: Dict[str, Any],
+        msg_id: str,
+        seq: int,
+    ) -> int:
+        """Append tool_use blocks from metadata tool_calls."""
+        tool_calls = metadata.get("tool_calls") or metadata.get("tool_call")
+        if tool_calls:
+            if isinstance(tool_calls, list):
+                for tc in tool_calls:
+                    if isinstance(tc, dict):
+                        name = (
+                            tc.get("name")
+                            or tc.get("function", {}).get("name")
+                            or "unknown"
+                        )
+                        args = (
+                            tc.get("args")
+                            or tc.get("arguments")
+                            or tc.get("function", {}).get("arguments")
+                        )
+                        blocks.append({
+                            "type": "tool_use",
+                            "name": name,
+                            "input": args,
+                            "seq": seq,
+                            "provider_message_id": msg_id,
+                        })
+                        seq += 1
+            elif isinstance(tool_calls, dict):
+                name = (
+                    tool_calls.get("name")
+                    or tool_calls.get("function", {}).get("name")
+                    or "unknown"
+                )
+                args = (
+                    tool_calls.get("args")
+                    or tool_calls.get("arguments")
+                    or tool_calls.get("function", {}).get("arguments")
+                )
+                blocks.append({
+                    "type": "tool_use",
+                    "name": name,
+                    "input": args,
+                    "seq": seq,
+                    "provider_message_id": msg_id,
+                })
+                seq += 1
+        return seq
+
+    def _append_text(
+        self, blocks: List[ContentBlock], content: Any, seq: int
+    ) -> int:
+        """Append a text block from regular content, if any."""
+        text = self._extract_text(content)
+        if text:
+            blocks.append({
+                "type": "text",
+                "text": text,
+                "seq": seq,
+            })
+            seq += 1
+        return seq
+
+    def _append_images(
+        self, blocks: List[ContentBlock], content: Any, seq: int
+    ) -> int:
+        """Append image blocks from multimodal content parts."""
+        if isinstance(content, dict):
+            parts = content.get("parts", [])
+            for part in parts:
+                if isinstance(part, dict):
+                    if part.get("content_type", "").startswith("image/"):
+                        blocks.append({
+                            "type": "image",
+                            "asset_pointer": part.get("asset_pointer"),
+                            "mime_type": part.get("content_type"),
+                            "seq": seq,
+                        })
+                        seq += 1
+        return seq
+
+    def _extract_text(self, content: Any) -> str:
+        """Extract text from various content formats."""
+        if isinstance(content, str):
+            return content
+        if isinstance(content, dict):
+            if "text" in content:
+                return content["text"]
+            parts = content.get("parts", [])
+            text_parts = []
+            for part in parts:
+                if isinstance(part, str):
+                    text_parts.append(part)
+                elif isinstance(part, dict) and "text" in part:
+                    text_parts.append(part["text"])
+            return "\n\n".join(text_parts)
+        return ""