thread-archive 0.0.2__py3-none-any.whl

thread_archive/_thread_import/parsers/types/cursor.py

@@ -0,0 +1,109 @@
+"""
+TypedDict schemas for Cursor IDE chat export format.
+
+Storage location (local machine, not devcontainers):
+- macOS: ~/Library/Application Support/Cursor/User/workspaceStorage/
+- Linux: ~/.config/Cursor/User/workspaceStorage/
+- Windows: %APPDATA%\\Cursor\\User\\workspaceStorage\\
+
+Each workspace has a hashed subdirectory containing state.vscdb (SQLite).
+
+Export methods:
+1. cursor-export CLI tool (recommended) -> ZIP with cursor_chats.json
+2. Cursor UI: Chat menu -> Export Chat -> Markdown file (single chat only)
+
+Key characteristics:
+- Multiple format versions (v1 simple, v2 structured)
+- Role normalization needed (human/ai/bot -> user/assistant)
+- Timestamps may be ISO string or Unix (seconds/milliseconds)
+- Linear conversations (no branching)
+"""
+
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+
+
+class CursorContentBlock(TypedDict, total=False):
+    """A content block in a Cursor message.
+
+    Block types:
+    - text: Plain text content
+    - tool_use: Tool/function call
+    - tool_result: Tool/function result
+    - thinking: Extended thinking (model-dependent)
+    - code: Code block with language
+    """
+    type: str  # "text", "tool_use", "tool_result", "thinking", "code"
+    text: Optional[str]  # For text and thinking blocks
+    name: Optional[str]  # For tool_use and tool_result
+    input: Optional[Dict[str, Any]]  # For tool_use
+    content: Optional[Any]  # For tool_result
+    is_error: Optional[bool]  # For tool_result
+    code: Optional[str]  # For code blocks
+    language: Optional[str]  # For code blocks
+
+
+class CursorToolCall(TypedDict, total=False):
+    """Tool call structure in Cursor messages."""
+    name: str
+    arguments: Optional[Union[str, Dict[str, Any]]]
+    input: Optional[Dict[str, Any]]
+    args: Optional[Union[str, Dict[str, Any]]]
+    call_id: Optional[str]
+    result: Optional[Any]
+    status: Optional[str]
+    user_decision: Optional[str]
+    function: Optional[Dict[str, Any]]  # OpenAI-style nesting
+
+
+class CursorCodeBlock(TypedDict, total=False):
+    """Code block from v2 export format (cursor_export.py output)."""
+    uri: Optional[Dict[str, Any]]  # {path: str, ...}
+    version: Optional[int]
+    codeBlockIdx: Optional[int]
+    content: str
+    languageId: Optional[str]
+    language: Optional[str]
+
+
+class CursorMessage(TypedDict, total=False):
+    """A single message in a Cursor conversation."""
+    id: Optional[str]
+    role: str  # "user", "human", "assistant", "ai", "bot", "system", "tool"
+    content: Union[str, List[Any]]  # String or array of content items
+    content_blocks: Optional[List[CursorContentBlock]]
+    created_at: Optional[Union[str, int, float]]  # ISO or Unix timestamp
+    model: Optional[str]
+
+    # Tool-related fields
+    tool_calls: Optional[List[CursorToolCall]]
+    toolCalls: Optional[List[CursorToolCall]]  # Alternative casing
+    tool_results: Optional[List[Dict[str, Any]]]
+    toolResults: Optional[List[Dict[str, Any]]]  # Alternative casing
+    tool_call: Optional[CursorToolCall]  # v2 single tool call
+
+    # Thinking
+    thinking: Optional[Union[str, List[Any]]]
+    reasoning: Optional[Union[str, List[Any]]]
+    thinking_duration_ms: Optional[int]
+
+    # Code blocks (v2 format)
+    code_blocks: Optional[List[CursorCodeBlock]]
+
+
+class CursorConversation(TypedDict, total=False):
+    """A single Cursor conversation."""
+    id: str
+    title: Optional[str]
+    created_at: Optional[Union[str, int, float]]
+    updated_at: Optional[Union[str, int, float]]
+    workspace_hash: Optional[str]
+    workspace_name: Optional[str]
+    workspace_uri: Optional[str]
+    messages: List[CursorMessage]
+
+
+class CursorExport(TypedDict, total=False):
+    """Cursor export bundle from cursor-export CLI tool."""
+    provider: Literal["cursor"]
+    conversations: List[CursorConversation]
+    export_metadata: Optional[Dict[str, Any]]

thread_archive/_thread_import/parsers/validators/__init__.py

@@ -0,0 +1,83 @@
+"""
+Pluggable validators for the parser pipeline.
+
+Validators check messages against rules and report errors/warnings.
+Each validator focuses on a single concern for testability.
+"""
+
+import logging
+
+from ..config import ProviderConfig, get_provider_config
+from .base import BaseValidator, ValidationContext
+from .content import ContentValidator
+from .referential import ReferentialIntegrityValidator
+from .thinking import ThinkingBlockValidator
+from .types import TypeValidator
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "ValidationContext",
+    "BaseValidator",
+    "ThinkingBlockValidator",
+    "ReferentialIntegrityValidator",
+    "TypeValidator",
+    "ContentValidator",
+    "validate_messages",
+]
+
+# Run order: content/type first (per-message shape), then thinking/referential
+# (aggregate). All are read-only — they only collect issues into the context.
+_VALIDATORS = (
+    ContentValidator,
+    TypeValidator,
+    ThinkingBlockValidator,
+    ReferentialIntegrityValidator,
+)
+
+
+# Validators with no cross-message / conversation-aggregate state — safe to run
+# on a *partial* conversation (an incremental import batch) without false
+# positives. TypeValidator only scans each block/role against the known set, so
+# it catches provider format drift (a new block type / role) on any slice.
+_BATCH_SAFE_VALIDATORS = (TypeValidator,)
+
+
+def validate_messages(messages, conversation_id, source_provider, *, strict=False, batch_safe=False):
+    """Run the validators over one parsed/assembled conversation and return the
+    ValidationContext (``.errors`` / ``.warnings``). Pure — no I/O; the caller
+    decides what to do with the issues (we log them, never reject).
+
+    An unknown provider falls back to a permissive default config, so the
+    universal checks (unknown block types, missing content/timestamps) still run
+    everywhere; provider-specific checks (thinking expectation, expected roles)
+    only kick in for the configured providers.
+
+    ``batch_safe=True`` runs only the validators with no conversation-aggregate
+    state (just format-drift type checks), so it's correct on a *partial*
+    conversation — used by the incremental watcher path, where each poll imports
+    only a slice and the aggregate checks ("0% thinking", "no user messages")
+    would otherwise false-fire.
+    """
+    try:
+        config = get_provider_config(source_provider)
+    except KeyError:
+        config = ProviderConfig(provider_name=source_provider)
+    context = ValidationContext(
+        conversation_id=conversation_id or "",
+        source_provider=source_provider,
+        strict=strict,
+    )
+    for validator_cls in (_BATCH_SAFE_VALIDATORS if batch_safe else _VALIDATORS):
+        try:
+            validator_cls(config, strict=strict).validate(messages, context)
+        except Exception:  # one bad validator must never break an import
+            # ...but it must never be silent either: surface the crash loudly
+            # (with traceback) while still letting the import proceed.
+            logger.exception(
+                "Validator %s crashed for conversation %r (provider %r); skipping it",
+                validator_cls.__name__,
+                conversation_id,
+                source_provider,
+            )
+    return context

thread_archive/_thread_import/parsers/validators/base.py

@@ -0,0 +1,168 @@
+"""
+Base validation infrastructure.
+
+Provides ValidationContext and BaseValidator for building validators.
+"""
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import List, Optional
+
+from ..base import NormalizedMessage
+from ..config import ProviderConfig
+
+
+class ValidationSeverity(str, Enum):
+    """Severity level for validation issues."""
+    error = "error"
+    warning = "warning"
+
+
+@dataclass
+class ValidationContext:
+    """Context for validation across a conversation.
+
+    Tracks state as messages are processed and collects errors/warnings.
+    """
+
+    conversation_id: str
+    source_provider: str
+    strict: bool = False  # If True, warnings become errors
+
+    # Counters
+    message_count: int = 0
+    first_user_msg_seen: bool = False
+    first_user_msg_has_context: bool = False
+    assistant_msg_count: int = 0
+    assistant_msgs_with_thinking: int = 0
+    assistant_msgs_with_tools: int = 0
+
+    # Model-specific tracking
+    thinking_required_msgs: int = 0
+    thinking_required_msgs_with_thinking: int = 0
+
+    # Results
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+
+    def add_error(self, msg: str) -> None:
+        """Add a validation error."""
+        self.errors.append(msg)
+
+    def add_warning(self, msg: str) -> None:
+        """Add a validation warning."""
+        if self.strict:
+            self.errors.append(msg)
+        else:
+            self.warnings.append(msg)
+
+    def add_issue(self, msg: str, severity: ValidationSeverity) -> None:
+        """Add an issue with specified severity."""
+        if severity == ValidationSeverity.error:
+            self.add_error(msg)
+        else:
+            self.add_warning(msg)
+
+    @property
+    def has_errors(self) -> bool:
+        """Check if any errors were recorded."""
+        return len(self.errors) > 0
+
+
+class BaseValidator:
+    """Base class for validators with common utilities.
+
+    Subclasses should override validate() and optionally
+    validate_conversation_end().
+    """
+
+    def __init__(self, config: ProviderConfig, strict: bool = False):
+        """Initialize validator.
+
+        Args:
+            config: Provider-specific configuration
+            strict: If True, treat warnings as errors
+        """
+        self.config = config
+        self.strict = strict
+
+    def validate(
+        self, messages: List[NormalizedMessage], context: ValidationContext
+    ) -> None:
+        """Validate messages and update context.
+
+        Override in subclasses to implement validation logic.
+
+        Args:
+            messages: Messages to validate
+            context: Validation context to update
+        """
+        pass
+
+    def validate_message(
+        self, msg: NormalizedMessage, context: ValidationContext
+    ) -> None:
+        """Validate a single message.
+
+        Override in subclasses for per-message validation.
+
+        Args:
+            msg: Message to validate
+            context: Validation context to update
+        """
+        pass
+
+    def validate_conversation_end(self, context: ValidationContext) -> None:
+        """Perform end-of-conversation validation.
+
+        Override in subclasses for aggregate validation.
+
+        Args:
+            context: Validation context to update
+        """
+        pass
+
+    def _add_issue(
+        self, context: ValidationContext, msg: str, severity: ValidationSeverity
+    ) -> None:
+        """Add an issue, respecting strict mode."""
+        if self.strict and severity == ValidationSeverity.warning:
+            severity = ValidationSeverity.error
+        context.add_issue(msg, severity)
+
+    @staticmethod
+    def _has_thinking_blocks(msg: NormalizedMessage) -> bool:
+        """Check if message has thinking/reasoning blocks."""
+        blocks = msg.get("content_blocks", [])
+        return any(b.get("type") == "thinking" for b in blocks)
+
+    @staticmethod
+    def _has_tool_blocks(msg: NormalizedMessage) -> bool:
+        """Check if message has tool use or result blocks."""
+        blocks = msg.get("content_blocks", [])
+        return any(b.get("type") in ("tool_use", "tool_result") for b in blocks)
+
+    @staticmethod
+    def _has_user_context(msg: NormalizedMessage) -> bool:
+        """Check if message has user context blocks."""
+        blocks = msg.get("content_blocks", [])
+        if any(b.get("type") == "system_context" for b in blocks):
+            return True
+        # Check provider_data for context indicators
+        provider_data = msg.get("provider_data", {})
+        raw_msg = provider_data.get("message", {})
+        metadata = raw_msg.get("metadata", {})
+        return bool(metadata.get("is_user_system_message"))
+
+    @staticmethod
+    def _get_model(msg: NormalizedMessage) -> Optional[str]:
+        """Extract model name from message."""
+        provider_data = msg.get("provider_data", {})
+        # ChatGPT stores in message.metadata.model_slug
+        raw_msg = provider_data.get("message", {})
+        metadata = raw_msg.get("metadata", {})
+        model = metadata.get("model_slug") or metadata.get("model")
+        if model:
+            return model
+        # Check top-level provider_data
+        return provider_data.get("model")

thread_archive/_thread_import/parsers/validators/content.py

@@ -0,0 +1,80 @@
+"""
+Content validator.
+
+Validates message content presence and timestamps.
+"""
+
+from typing import List
+
+from ..base import NormalizedMessage
+from .base import BaseValidator, ValidationContext, ValidationSeverity
+
+
+class ContentValidator(BaseValidator):
+    """Validates message content and metadata.
+
+    Checks:
+    - Messages have content_text or content_blocks
+    - Messages have timestamps
+    - First user message tracking (for context validation)
+
+    Empty messages that aren't stubs are flagged as warnings.
+    """
+
+    def validate(
+        self, messages: List[NormalizedMessage], context: ValidationContext
+    ) -> None:
+        """Validate content across all messages."""
+        for msg in messages:
+            self._validate_message(msg, context)
+
+        # End-of-conversation check
+        self._validate_conversation_end(context)
+
+    def _validate_message(
+        self, msg: NormalizedMessage, context: ValidationContext
+    ) -> None:
+        """Validate a single message."""
+        context.message_count += 1
+        role = msg.get("role", "")
+
+        # Track first user message
+        if role == "user" and not context.first_user_msg_seen:
+            context.first_user_msg_seen = True
+            context.first_user_msg_has_context = self._has_user_context(msg)
+
+        # Check for content
+        has_text = bool(msg.get("content_text"))
+        has_blocks = bool(msg.get("content_blocks"))
+
+        if not has_text and not has_blocks:
+            # Skip validation for stub messages (ChatGPT branching artifacts)
+            provider_data = msg.get("provider_data", {})
+            is_stub = provider_data.get("synthetic_stub") or not provider_data.get(
+                "message"
+            )
+            if not is_stub:
+                self._add_issue(
+                    context,
+                    f"Message {msg.get('provider_message_id', 'unknown')} "
+                    f"has no content_text or content_blocks",
+                    ValidationSeverity.warning,
+                )
+
+        # Check for timestamp
+        if not msg.get("created_at"):
+            self._add_issue(
+                context,
+                f"Message {msg.get('provider_message_id', 'unknown')} lacks created_at timestamp",
+                ValidationSeverity.warning,
+            )
+
+    def _validate_conversation_end(self, context: ValidationContext) -> None:
+        """Perform end-of-conversation validation."""
+        # Check if we saw any user messages
+        if not context.first_user_msg_seen and context.message_count > 0:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id} has no user messages",
+                ValidationSeverity.warning,
+            )

thread_archive/_thread_import/parsers/validators/referential.py

@@ -0,0 +1,57 @@
+"""
+Referential integrity validator.
+
+Validates that parent references resolve to existing messages.
+"""
+
+from typing import List, Set
+
+from ..base import NormalizedMessage
+from .base import BaseValidator, ValidationContext, ValidationSeverity
+
+# Known root sentinel values that aren't real missing parents
+ROOT_SENTINELS = {"", "root", "aaa", "00000000-0000-0000-0000-000000000000"}
+
+
+class ReferentialIntegrityValidator(BaseValidator):
+    """Validates parent message references.
+
+    Missing parent references are common and expected in:
+    - ChatGPT: Branching exports may only include active path
+    - Claude Code: Context summarization removes old messages
+      but keeps parentUuid references
+
+    These emit warnings, not errors, since data is still usable.
+    """
+
+    def validate(
+        self, messages: List[NormalizedMessage], context: ValidationContext
+    ) -> None:
+        """Validate parent references resolve."""
+        if not self.config.has_parent_references:
+            # Provider doesn't use parent references
+            return
+
+        # Build set of all message IDs
+        message_ids: Set[str] = set()
+        for msg in messages:
+            msg_id = msg.get("provider_message_id")
+            if msg_id:
+                message_ids.add(msg_id)
+
+        # Count orphaned references
+        orphan_count = 0
+        for msg in messages:
+            parent_id = msg.get("provider_parent_id")
+            if parent_id and parent_id not in message_ids:
+                # Skip root sentinels
+                if parent_id not in ROOT_SENTINELS:
+                    orphan_count += 1
+
+        if orphan_count > 0:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id}: {orphan_count} message(s) reference "
+                f"missing parents (expected for branched/summarized conversations)",
+                ValidationSeverity.warning,
+            )

thread_archive/_thread_import/parsers/validators/thinking.py

@@ -0,0 +1,143 @@
+"""
+Thinking block validator.
+
+Validates thinking block presence based on provider expectations
+and model-specific requirements.
+"""
+
+from typing import List
+
+from ..base import NormalizedMessage
+from .base import BaseValidator, ValidationContext, ValidationSeverity
+
+
+class ThinkingBlockValidator(BaseValidator):
+    """Validates thinking block presence.
+
+    Uses ProviderConfig to determine expectations:
+    - "never": Error if thinking blocks exist (wrong parser/source)
+    - "required": Error if no thinking blocks (incomplete export)
+    - "model_specific": Validate per-message based on model
+    - "optional": No validation
+
+    This is a critical validator for catching:
+    1. Data parsed with wrong provider
+    2. Incomplete exports missing thinking data
+    """
+
+    def validate(
+        self, messages: List[NormalizedMessage], context: ValidationContext
+    ) -> None:
+        """Validate thinking blocks across all messages."""
+        # First pass: count and track per-message
+        for msg in messages:
+            self._track_message(msg, context)
+
+        # Second pass: aggregate validation
+        self._validate_aggregate(context)
+
+    def _track_message(
+        self, msg: NormalizedMessage, context: ValidationContext
+    ) -> None:
+        """Track thinking block presence for a message."""
+        role = msg.get("role", "")
+        if role != "assistant":
+            return
+
+        context.assistant_msg_count += 1
+        has_thinking = self._has_thinking_blocks(msg)
+
+        if has_thinking:
+            context.assistant_msgs_with_thinking += 1
+
+        # Track model-specific requirements
+        if self.config.thinking_expectation == "model_specific":
+            model = self._get_model(msg)
+            if self.config.requires_thinking(model):
+                context.thinking_required_msgs += 1
+                if has_thinking:
+                    context.thinking_required_msgs_with_thinking += 1
+
+    def _validate_aggregate(self, context: ValidationContext) -> None:
+        """Perform aggregate validation at conversation end."""
+        if context.assistant_msg_count == 0:
+            return
+
+        thinking_pct = (
+            context.assistant_msgs_with_thinking / context.assistant_msg_count * 100
+        )
+
+        expectation = self.config.thinking_expectation
+
+        if expectation == "never":
+            self._validate_never(context, thinking_pct)
+        elif expectation == "required":
+            self._validate_required(context, thinking_pct)
+        elif expectation == "model_specific":
+            self._validate_model_specific(context)
+
+    @staticmethod
+    def _is_subagent(context: ValidationContext) -> bool:
+        """Subagent sessions (agent-*) use Haiku without extended thinking."""
+        return bool(
+            context.conversation_id
+            and context.conversation_id.startswith("agent-")
+        )
+
+    def _validate_never(
+        self, context: ValidationContext, thinking_pct: float
+    ) -> None:
+        """``never``: any thinking blocks signal the wrong parser/source."""
+        if thinking_pct > 0:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id}: {self.config.provider_name} exports "
+                f"should NEVER have thinking blocks (found {thinking_pct:.0f}%). "
+                f"Wrong parser or data source?",
+                ValidationSeverity.error,
+            )
+
+    def _validate_required(
+        self, context: ValidationContext, thinking_pct: float
+    ) -> None:
+        """``required``: every (non-subagent) assistant turn must have thinking."""
+        if self._is_subagent(context):
+            return
+        if thinking_pct == 0:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id}: {self.config.provider_name} exports "
+                f"MUST have thinking blocks (found 0%). Incomplete export.",
+                ValidationSeverity.error,
+            )
+        elif thinking_pct < 100:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id}: {thinking_pct:.0f}% of "
+                f"assistant messages have thinking blocks (expected 100%)",
+                ValidationSeverity.warning,
+            )
+
+    def _validate_model_specific(self, context: ValidationContext) -> None:
+        """``model_specific``: only thinking-capable model turns are checked."""
+        if self._is_subagent(context) or context.thinking_required_msgs <= 0:
+            return
+        model_thinking_pct = (
+            context.thinking_required_msgs_with_thinking
+            / context.thinking_required_msgs
+            * 100
+        )
+        if model_thinking_pct == 0:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id}: thinking-capable model messages "
+                f"should have thinking blocks (found 0%). Incomplete export.",
+                ValidationSeverity.error,
+            )
+        elif model_thinking_pct < 100:
+            self._add_issue(
+                context,
+                f"Conversation {context.conversation_id}: {model_thinking_pct:.0f}% of "
+                f"thinking-capable model messages have thinking blocks",
+                ValidationSeverity.warning,
+            )