thread-archive 0.0.2__py3-none-any.whl

thread_archive/_thread_import/parsers/validators/types.py

@@ -0,0 +1,87 @@
+"""
+Type validator.
+
+Validates roles and content block types against known values.
+"""
+
+from typing import List, Set
+
+from ..base import NormalizedMessage
+from .base import BaseValidator, ValidationContext, ValidationSeverity
+
+# Universal known block types across all providers
+KNOWN_BLOCK_TYPES: Set[str] = {
+    "text",
+    "thinking",
+    "tool_use",
+    "tool_result",
+    "image",
+    "file",
+    "code",
+    "system_context",
+    "parse_error",
+    "file_snapshot",
+    "context_summary",
+    "system_metadata",
+    "ide_context",
+    "queue_operation",
+    "progress",
+}
+
+
+class TypeValidator(BaseValidator):
+    """Validates message roles and content block types.
+
+    Unknown types get warnings (not errors) because:
+    - Provider may have added new features
+    - We want to preserve unknown data, not reject it
+
+    This helps detect format changes in provider exports.
+    """
+
+    def validate(
+        self, messages: List[NormalizedMessage], context: ValidationContext
+    ) -> None:
+        """Validate types across all messages."""
+        unknown_roles: Set[str] = set()
+        unknown_block_types: Set[str] = set()
+
+        for msg in messages:
+            role = msg.get("role", "")
+
+            # Check role against provider's expected roles
+            if role and self.config.expected_roles:
+                if role not in self.config.expected_roles:
+                    unknown_roles.add(role)
+
+            # Check content block types
+            for block in msg.get("content_blocks", []):
+                block_type = block.get("type", "")
+                if block_type:
+                    # Check against universal known types
+                    if block_type not in KNOWN_BLOCK_TYPES:
+                        unknown_block_types.add(block_type)
+                    # Optionally check against provider-specific expected types
+                    elif (
+                        self.config.expected_block_types
+                        and block_type not in self.config.expected_block_types
+                    ):
+                        # Provider has restricted set - warn about unexpected
+                        unknown_block_types.add(block_type)
+
+        # Report unknown types
+        for role in sorted(unknown_roles):
+            self._add_issue(
+                context,
+                f"Unknown message role '{role}' from provider {self.config.provider_name} - "
+                f"possible format change or new feature",
+                ValidationSeverity.warning,
+            )
+
+        for block_type in sorted(unknown_block_types):
+            self._add_issue(
+                context,
+                f"Unknown content block type '{block_type}' - "
+                f"possible format change or new feature",
+                ValidationSeverity.warning,
+            )

thread_archive/_thread_import/schemas/__init__.py

@@ -0,0 +1,231 @@
+"""
+JSON Schema validation for provider export formats.
+
+Provides versioned schema validation with automatic version detection.
+Schemas are stored as JSON files in provider-specific subdirectories.
+
+Usage:
+    from apps.chat_import.schemas import validate_export, detect_schema_version
+
+    # Auto-detect and validate
+    errors, warnings = validate_export(data, provider="chatgpt")
+
+    # Or detect version explicitly
+    version = detect_schema_version(data, provider="chatgpt")
+    errors, warnings = validate_export(data, provider="chatgpt", version=version)
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+try:
+    import jsonschema
+    HAS_JSONSCHEMA = True
+except ImportError:
+    HAS_JSONSCHEMA = False
+
+
+SCHEMA_DIR = Path(__file__).parent
+
+
+def get_available_versions(provider: str) -> List[str]:
+    """Get list of available schema versions for a provider."""
+    provider_dir = SCHEMA_DIR / provider
+    if not provider_dir.exists():
+        return []
+
+    versions = []
+    for schema_file in provider_dir.glob("v*.json"):
+        versions.append(schema_file.stem)  # e.g., "v1", "v2"
+
+    return sorted(versions)
+
+
+def load_schema(provider: str, version: str) -> Optional[Dict[str, Any]]:
+    """Load a specific schema version."""
+    schema_path = SCHEMA_DIR / provider / f"{version}.json"
+    if not schema_path.exists():
+        return None
+
+    with open(schema_path) as f:
+        return json.load(f)
+
+
+def detect_schema_version(data: Any, provider: str) -> str:
+    """
+    Auto-detect schema version from data structure.
+
+    Uses heuristics specific to each provider to determine which
+    schema version the data matches.
+
+    Returns the detected version string (e.g., "v1", "v2") or "v1" as default.
+    """
+    if provider == "chatgpt":
+        return _detect_chatgpt_version(data)
+    elif provider == "claude":
+        return _detect_claude_version(data)
+    elif provider == "claude-code":
+        return _detect_claude_code_version(data)
+    elif provider == "cursor":
+        return _detect_cursor_version(data)
+
+    return "v1"
+
+
+def _detect_chatgpt_version(data: Any) -> str:
+    """Detect ChatGPT export version.
+
+    v2: Has thinking blocks (content_type: "thoughts")
+    v1: Original format
+    """
+    if not isinstance(data, list):
+        data = [data]
+
+    for conv in data:
+        mapping = conv.get("mapping", {})
+        for node in mapping.values():
+            msg = node.get("message")
+            if msg:
+                metadata = msg.get("metadata", {})
+                content_type = metadata.get("content_type")
+                if content_type in ("thoughts", "analysis", "reasoning_recap"):
+                    return "v2"
+
+    return "v1"
+
+
+def _detect_claude_version(data: Any) -> str:
+    """Detect Claude export version.
+
+    v1: Current format (only version so far)
+    """
+    return "v1"
+
+
+def _detect_claude_code_version(data: Any) -> str:
+    """Detect Claude Code session version.
+
+    v1: Current JSONL format (only version so far)
+    """
+    return "v1"
+
+
+def _detect_cursor_version(data: Any) -> str:
+    """Detect Cursor export version.
+
+    v2: Has toolFormerData field
+    v1: Original format
+    """
+    if isinstance(data, dict):
+        bubbles = data.get("bubbles", [])
+    elif isinstance(data, list):
+        bubbles = data
+    else:
+        return "v1"
+
+    for bubble in bubbles:
+        if bubble.get("toolFormerData"):
+            return "v2"
+
+    return "v1"
+
+
+def validate_export(
+    data: Any,
+    provider: str,
+    version: Optional[str] = None,
+) -> Tuple[List[str], List[str]]:
+    """
+    Validate export data against JSON Schema.
+
+    Args:
+        data: The export data to validate
+        provider: Provider name (chatgpt, claude, claude-code, cursor)
+        version: Schema version (auto-detected if not specified)
+
+    Returns:
+        Tuple of (errors, warnings) where:
+        - errors: List of validation error messages
+        - warnings: List of validation warning messages (unknown fields, etc.)
+    """
+    if not HAS_JSONSCHEMA:
+        # Return empty if jsonschema not installed
+        return [], ["jsonschema library not installed, skipping validation"]
+
+    if version is None:
+        version = detect_schema_version(data, provider)
+
+    schema = load_schema(provider, version)
+    if schema is None:
+        # No schema file for this version - not an error, just a warning
+        available = get_available_versions(provider)
+        if available:
+            return [], [f"No schema for {provider} {version}, available: {available}"]
+        else:
+            return [], [f"No schemas defined for provider {provider}"]
+
+    errors: List[str] = []
+    warnings: List[str] = []
+
+    try:
+        # Validate against schema
+        validator = jsonschema.Draft7Validator(schema)
+
+        for error in validator.iter_errors(data):
+            # Categorize errors
+            if error.validator == "additionalProperties":
+                # Unknown field - warning not error
+                warnings.append(f"Unknown field at {_format_path(error.path)}: {error.message}")
+            elif error.validator == "required":
+                # Missing required field - error
+                errors.append(f"Missing required field at {_format_path(error.path)}: {error.message}")
+            elif error.validator == "type":
+                # Type mismatch - error
+                errors.append(f"Type mismatch at {_format_path(error.path)}: {error.message}")
+            else:
+                # Other validation errors
+                errors.append(f"Validation error at {_format_path(error.path)}: {error.message}")
+
+    except Exception as e:
+        errors.append(f"Schema validation failed: {str(e)}")
+
+    return errors, warnings
+
+
+def _format_path(path) -> str:
+    """Format a JSON path for display."""
+    if not path:
+        return "root"
+    parts = []
+    for p in path:
+        if isinstance(p, int):
+            parts.append(f"[{p}]")
+        else:
+            parts.append(f".{p}")
+    return "".join(parts).lstrip(".")
+
+
+def get_schema_violations(
+    data: Any,
+    provider: str,
+    version: Optional[str] = None,
+) -> Dict[str, List[str]]:
+    """
+    Get detailed schema violations.
+
+    Returns:
+        Dict with keys:
+        - unknown_fields: Fields not in schema
+        - missing_fields: Required fields that are missing
+        - type_mismatches: Fields with wrong types
+        - other_errors: Other validation errors
+    """
+    errors, warnings = validate_export(data, provider, version)
+
+    return {
+        "unknown_fields": [w for w in warnings if "Unknown field" in w],
+        "missing_fields": [e for e in errors if "Missing required" in e],
+        "type_mismatches": [e for e in errors if "Type mismatch" in e],
+        "other_errors": [e for e in errors if "Missing required" not in e and "Type mismatch" not in e],
+    }

thread_archive/_thread_import/schemas/chatgpt/v1.json

@@ -0,0 +1,197 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "chatgpt_export_v1",
+  "title": "ChatGPT Export v1",
+  "description": "Original ChatGPT conversation export format",
+  "type": "array",
+  "items": {
+    "$ref": "#/definitions/conversation"
+  },
+  "definitions": {
+    "conversation": {
+      "type": "object",
+      "required": ["title", "mapping"],
+      "properties": {
+        "title": {
+          "type": ["string", "null"],
+          "description": "Conversation title"
+        },
+        "create_time": {
+          "type": ["number", "null"],
+          "description": "Unix timestamp when conversation was created"
+        },
+        "update_time": {
+          "type": ["number", "null"],
+          "description": "Unix timestamp when conversation was last updated"
+        },
+        "mapping": {
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "#/definitions/mappingNode"
+          },
+          "description": "Tree structure of messages"
+        },
+        "current_node": {
+          "type": ["string", "null"],
+          "description": "ID of the currently active message node"
+        },
+        "conversation_template_id": {
+          "type": ["string", "null"]
+        },
+        "gizmo_id": {
+          "type": ["string", "null"]
+        },
+        "id": {
+          "type": "string"
+        },
+        "moderation_results": {
+          "type": "array"
+        },
+        "safe_urls": {
+          "type": "array"
+        }
+      }
+    },
+    "mappingNode": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string",
+          "description": "Node/message ID"
+        },
+        "parent": {
+          "type": ["string", "null"],
+          "description": "Parent node ID (null for root)"
+        },
+        "children": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Child node IDs"
+        },
+        "message": {
+          "oneOf": [
+            {"$ref": "#/definitions/message"},
+            {"type": "null"}
+          ],
+          "description": "Message content (null for structural nodes)"
+        }
+      }
+    },
+    "message": {
+      "type": "object",
+      "required": ["id", "author", "content"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "description": "Message ID"
+        },
+        "author": {
+          "$ref": "#/definitions/author"
+        },
+        "content": {
+          "$ref": "#/definitions/content"
+        },
+        "create_time": {
+          "type": ["number", "null"],
+          "description": "Unix timestamp when message was created"
+        },
+        "update_time": {
+          "type": ["number", "null"]
+        },
+        "status": {
+          "type": "string",
+          "enum": ["finished_successfully", "in_progress", "finished_partial_completion"]
+        },
+        "end_turn": {
+          "type": ["boolean", "null"]
+        },
+        "weight": {
+          "type": "number"
+        },
+        "metadata": {
+          "$ref": "#/definitions/metadata"
+        },
+        "recipient": {
+          "type": "string"
+        }
+      }
+    },
+    "author": {
+      "type": "object",
+      "required": ["role"],
+      "properties": {
+        "role": {
+          "type": "string",
+          "enum": ["user", "assistant", "system", "tool"],
+          "description": "Message author role"
+        },
+        "name": {
+          "type": ["string", "null"]
+        },
+        "metadata": {
+          "type": "object"
+        }
+      }
+    },
+    "content": {
+      "type": "object",
+      "properties": {
+        "content_type": {
+          "type": "string",
+          "enum": ["text", "code", "execution_output", "tether_browsing_display", "tether_quote", "multimodal_text", "system_error"]
+        },
+        "parts": {
+          "type": "array",
+          "description": "Content parts (text strings or objects)"
+        },
+        "language": {
+          "type": "string"
+        },
+        "text": {
+          "type": "string"
+        },
+        "result": {
+          "type": "string"
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "is_user_system_message": {
+          "type": "boolean",
+          "description": "True if this is user's custom instructions"
+        },
+        "user_context_message_data": {
+          "type": "object"
+        },
+        "model_slug": {
+          "type": "string"
+        },
+        "default_model_slug": {
+          "type": "string"
+        },
+        "finish_details": {
+          "type": "object"
+        },
+        "citations": {
+          "type": "array"
+        },
+        "gizmo_id": {
+          "type": ["string", "null"]
+        },
+        "message_type": {
+          "type": ["string", "null"]
+        },
+        "request_id": {
+          "type": "string"
+        },
+        "timestamp_": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}