bot-knows 0.1.0__py3-none-any.whl

bot_knows/importers/chatgpt.py

@@ -0,0 +1,154 @@
+"""ChatGPT import adapter for bot_knows.
+
+This module provides an adapter for parsing ChatGPT export files.
+"""
+
+from typing import Any, override
+
+from bot_knows.importers.base import ChatImportAdapter
+from bot_knows.importers.registry import ImportAdapterRegistry
+from bot_knows.models.ingest import ChatIngest, IngestMessage
+
+
+@ImportAdapterRegistry.register
+class ChatGPTAdapter(ChatImportAdapter):
+    """Adapter for ChatGPT export format (conversations.json).
+
+    Parses the JSON export from ChatGPT's "Export data" feature.
+    The export contains a list of conversations, each with a mapping
+    of message nodes.
+
+    Export format example:
+        [
+            {
+                "id": "conversation-id",
+                "title": "Chat Title",
+                "create_time": 1704067200,
+                "mapping": {
+                    "node-id": {
+                        "message": {
+                            "author": {"role": "user"},
+                            "content": {"parts": ["Hello"]},
+                            "create_time": 1704067200
+                        }
+                    }
+                }
+            }
+        ]
+    """
+
+    @property
+    @override
+    def source_name(self) -> str:
+        return "chatgpt"
+
+    @override
+    def parse(self, raw_export: dict[str, Any]) -> list[ChatIngest]:
+        """Parse ChatGPT export format.
+
+        Args:
+            raw_export: Raw JSON data (list of conversations or dict with 'conversations' key)
+
+        Returns:
+            List of ChatIngest objects
+        """
+        chats: list[ChatIngest] = []
+
+        # Handle both formats: direct list or dict with 'conversations' key
+        if isinstance(raw_export, list):
+            conversations = raw_export
+        else:
+            conversations = raw_export.get("conversations", [])
+
+        for conv in conversations:
+            chat_ingest = self._parse_conversation(conv)
+            if chat_ingest and chat_ingest.has_messages:
+                chats.append(chat_ingest)
+
+        return chats
+
+    def _parse_conversation(self, conv: dict[str, Any]) -> ChatIngest | None:
+        """Parse a single conversation."""
+        conv_id = conv.get("id", "")
+        title = conv.get("title")
+        create_time = self._parse_timestamp(conv.get("create_time", 0))
+
+        messages: list[IngestMessage] = []
+        mapping = conv.get("mapping", {})
+
+        for node in mapping.values():
+            msg = self._parse_message_node(node, conv_id, create_time)
+            if msg:
+                messages.append(msg)
+
+        # Sort messages by timestamp
+        messages.sort(key=lambda m: m.timestamp)
+
+        return ChatIngest(
+            source="chatgpt",
+            imported_chat_timestamp=create_time,
+            title=title,
+            messages=messages,
+            provider="chatgpt",
+            conversation_id=conv_id,
+        )
+
+    def _parse_message_node(
+        self,
+        node: dict[str, Any],
+        conv_id: str,
+        default_timestamp: int,
+    ) -> IngestMessage | None:
+        """Parse a message node from the mapping."""
+        msg = node.get("message")
+        if not msg:
+            return None
+
+        # Get role
+        author = msg.get("author", {})
+        role = author.get("role", "")
+        if role not in ("user", "assistant", "system"):
+            return None
+
+        # Get content
+        content = self._extract_content(msg)
+        if not content:
+            return None
+
+        # Get timestamp
+        timestamp = self._parse_timestamp(msg.get("create_time"))
+        if timestamp == 0:
+            timestamp = default_timestamp
+
+        return IngestMessage(
+            role=role,  # type: ignore[arg-type]
+            content=content,
+            timestamp=timestamp,
+            chat_id=conv_id,
+        )
+
+    def _extract_content(self, msg: dict[str, Any]) -> str:
+        """Extract text content from message."""
+        content_obj = msg.get("content", {})
+
+        # Handle different content formats
+        if isinstance(content_obj, str):
+            return content_obj
+
+        if isinstance(content_obj, dict):
+            parts = content_obj.get("parts", [])
+            # Filter and join text parts
+            text_parts = [str(p) for p in parts if p and isinstance(p, (str, int, float))]
+            return " ".join(text_parts).strip()
+
+        return ""
+
+    def _parse_timestamp(self, value: Any) -> int:
+        """Parse timestamp to epoch seconds."""
+        if value is None:
+            return 0
+        if isinstance(value, int):
+            return value
+        if isinstance(value, float):
+            return int(value)
+        return 0

bot_knows/importers/claude.py

@@ -0,0 +1,172 @@
+"""Claude import adapter for bot_knows.
+
+This module provides an adapter for parsing Claude export files.
+"""
+
+from typing import Any, override
+
+from bot_knows.importers.base import ChatImportAdapter
+from bot_knows.importers.registry import ImportAdapterRegistry
+from bot_knows.models.ingest import ChatIngest, IngestMessage
+
+
+@ImportAdapterRegistry.register
+class ClaudeAdapter(ChatImportAdapter):
+    """Adapter for Claude export format.
+
+    Parses the JSON export from Claude's export feature.
+    The exact format may vary based on Claude version.
+
+    Expected format:
+        {
+            "conversations": [
+                {
+                    "uuid": "conversation-id",
+                    "name": "Chat Title",
+                    "created_at": "2024-01-01T00:00:00Z",
+                    "chat_messages": [
+                        {
+                            "uuid": "message-id",
+                            "sender": "human",
+                            "text": "Hello",
+                            "created_at": "2024-01-01T00:00:00Z"
+                        }
+                    ]
+                }
+            ]
+        }
+    """
+
+    @property
+    @override
+    def source_name(self) -> str:
+        return "claude"
+
+    @override
+    def parse(self, raw_export: dict[str, Any]) -> list[ChatIngest]:
+        """Parse Claude export format.
+
+        Args:
+            raw_export: Raw JSON data
+
+        Returns:
+            List of ChatIngest objects
+        """
+        chats: list[ChatIngest] = []
+
+        # Handle different possible formats
+        if isinstance(raw_export, list):
+            conversations = raw_export
+        else:
+            conversations = raw_export.get("conversations", [])
+
+        for conv in conversations:
+            chat_ingest = self._parse_conversation(conv)
+            if chat_ingest and chat_ingest.has_messages:
+                chats.append(chat_ingest)
+
+        return chats
+
+    def _parse_conversation(self, conv: dict[str, Any]) -> ChatIngest | None:
+        """Parse a single conversation."""
+        conv_id = conv.get("uuid", conv.get("id", ""))
+        title = conv.get("name", conv.get("title"))
+
+        # Parse timestamp
+        created_at = conv.get("created_at", conv.get("create_time", 0))
+        create_time = self._parse_timestamp(created_at)
+
+        # Get messages
+        raw_messages = conv.get("chat_messages", conv.get("messages", []))
+        messages: list[IngestMessage] = []
+
+        for raw_msg in raw_messages:
+            msg = self._parse_message(raw_msg, conv_id, create_time)
+            if msg:
+                messages.append(msg)
+
+        # Sort messages by timestamp
+        messages.sort(key=lambda m: m.timestamp)
+
+        return ChatIngest(
+            source="claude",
+            imported_chat_timestamp=create_time,
+            title=title,
+            messages=messages,
+            provider="claude",
+            conversation_id=conv_id,
+        )
+
+    def _parse_message(
+        self,
+        raw_msg: dict[str, Any],
+        conv_id: str,
+        default_timestamp: int,
+    ) -> IngestMessage | None:
+        """Parse a single message."""
+        # Get role (Claude uses 'sender' with 'human'/'assistant')
+        sender = raw_msg.get("sender", raw_msg.get("role", ""))
+        role = self._normalize_role(sender)
+        if not role:
+            return None
+
+        # Get content
+        content = raw_msg.get("text", raw_msg.get("content", ""))
+        if isinstance(content, list):
+            # Handle content blocks format
+            content = self._extract_text_from_blocks(content)
+        if not content:
+            return None
+
+        # Get timestamp
+        created_at = raw_msg.get("created_at", raw_msg.get("timestamp"))
+        timestamp = self._parse_timestamp(created_at)
+        if timestamp == 0:
+            timestamp = default_timestamp
+
+        return IngestMessage(
+            role=role,  # type: ignore[arg-type]
+            content=content,
+            timestamp=timestamp,
+            chat_id=conv_id,
+        )
+
+    def _normalize_role(self, sender: str) -> str | None:
+        """Normalize sender/role to standard roles."""
+        sender_lower = sender.lower()
+        if sender_lower in ("human", "user"):
+            return "user"
+        if sender_lower in ("assistant", "ai", "claude"):
+            return "assistant"
+        if sender_lower == "system":
+            return "system"
+        return None
+
+    def _extract_text_from_blocks(self, blocks: list[Any]) -> str:
+        """Extract text from content blocks format."""
+        texts = []
+        for block in blocks:
+            if isinstance(block, str):
+                texts.append(block)
+            elif isinstance(block, dict) and block.get("type") == "text":
+                texts.append(block.get("text", ""))
+        return "\n".join(texts).strip()
+
+    def _parse_timestamp(self, value: Any) -> int:
+        """Parse timestamp to epoch seconds."""
+        if value is None:
+            return 0
+        if isinstance(value, int):
+            return value
+        if isinstance(value, float):
+            return int(value)
+        if isinstance(value, str):
+            # Try to parse ISO format
+            try:
+                from datetime import datetime
+
+                dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+                return int(dt.timestamp())
+            except (ValueError, TypeError):
+                return 0
+        return 0

bot_knows/importers/generic_json.py

@@ -0,0 +1,272 @@
+"""Generic JSON import adapter for bot_knows.
+
+This module provides a flexible adapter for custom JSON formats.
+"""
+
+from collections.abc import Callable
+from typing import Any, override
+
+from bot_knows.importers.base import ChatImportAdapter
+from bot_knows.importers.registry import ImportAdapterRegistry
+from bot_knows.models.ingest import ChatIngest, IngestMessage
+
+
+@ImportAdapterRegistry.register
+class GenericJSONAdapter(ChatImportAdapter):
+    """Flexible adapter for custom JSON formats.
+
+    This adapter can handle various JSON structures by allowing
+    customization of field mappings.
+
+    Expected default format:
+        {
+            "chats": [
+                {
+                    "id": "chat-id",
+                    "title": "Chat Title",
+                    "timestamp": 1704067200,
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": "Hello",
+                            "timestamp": 1704067200
+                        }
+                    ]
+                }
+            ]
+        }
+
+    Or simpler flat format:
+        [
+            {
+                "role": "user",
+                "content": "Hello",
+                "timestamp": 1704067200,
+                "chat_id": "chat-1"
+            }
+        ]
+    """
+
+    def __init__(
+        self,
+        chats_key: str = "chats",
+        messages_key: str = "messages",
+        role_key: str = "role",
+        content_key: str = "content",
+        timestamp_key: str = "timestamp",
+        chat_id_key: str = "id",
+        title_key: str = "title",
+        role_mapping: dict[str, str] | None = None,
+        content_extractor: Callable[[Any], str] | None = None,
+    ) -> None:
+        """Initialize adapter with field mappings.
+
+        Args:
+            chats_key: Key for chats array in root object
+            messages_key: Key for messages array in chat object
+            role_key: Key for role in message object
+            content_key: Key for content in message object
+            timestamp_key: Key for timestamp in message/chat object
+            chat_id_key: Key for chat ID in chat object
+            title_key: Key for title in chat object
+            role_mapping: Custom role mapping (e.g., {"human": "user"})
+            content_extractor: Custom function to extract content from message
+        """
+        self._chats_key = chats_key
+        self._messages_key = messages_key
+        self._role_key = role_key
+        self._content_key = content_key
+        self._timestamp_key = timestamp_key
+        self._chat_id_key = chat_id_key
+        self._title_key = title_key
+        self._role_mapping = role_mapping or {}
+        self._content_extractor = content_extractor
+
+    @property
+    @override
+    def source_name(self) -> str:
+        return "generic_json"
+
+    @override
+    def parse(self, raw_export: dict[str, Any]) -> list[ChatIngest]:
+        """Parse generic JSON format.
+
+        Supports:
+        1. Object with 'chats' array containing chat objects with 'messages'
+        2. Direct array of chat objects
+        3. Flat array of messages with chat_id field
+
+        Args:
+            raw_export: Raw JSON data
+
+        Returns:
+            List of ChatIngest objects
+        """
+        # Try different formats
+        if isinstance(raw_export, list):
+            return self._parse_list(raw_export)
+
+        if self._chats_key in raw_export:
+            return self._parse_chats_object(raw_export)
+
+        # Try to interpret as single chat
+        return self._parse_single_chat(raw_export)
+
+    def _parse_list(self, items: list[Any]) -> list[ChatIngest]:
+        """Parse list format (could be chats or flat messages)."""
+        if not items:
+            return []
+
+        # Check if first item looks like a chat or a message
+        first = items[0]
+        if isinstance(first, dict) and self._messages_key in first:
+            # List of chat objects
+            return [self._parse_chat(chat) for chat in items if chat]
+
+        # Assume flat list of messages - group by chat_id
+        return self._parse_flat_messages(items)
+
+    def _parse_chats_object(self, data: dict[str, Any]) -> list[ChatIngest]:
+        """Parse object with chats array."""
+        chats = data.get(self._chats_key, [])
+        return [self._parse_chat(chat) for chat in chats if chat]
+
+    def _parse_single_chat(self, data: dict[str, Any]) -> list[ChatIngest]:
+        """Parse as single chat object."""
+        chat = self._parse_chat(data)
+        return [chat] if chat.has_messages else []
+
+    def _parse_chat(self, chat: dict[str, Any]) -> ChatIngest:
+        """Parse a single chat object."""
+        chat_id = str(chat.get(self._chat_id_key, ""))
+        title = chat.get(self._title_key)
+        timestamp = self._parse_timestamp(chat.get(self._timestamp_key, 0))
+
+        raw_messages = chat.get(self._messages_key, [])
+        messages = [
+            msg
+            for msg in (self._parse_message(m, chat_id, timestamp) for m in raw_messages)
+            if msg is not None
+        ]
+
+        messages.sort(key=lambda m: m.timestamp)
+
+        return ChatIngest(
+            source="generic_json",
+            imported_chat_timestamp=timestamp,
+            title=title,
+            messages=messages,
+            conversation_id=chat_id,
+        )
+
+    def _parse_flat_messages(self, messages: list[Any]) -> list[ChatIngest]:
+        """Parse flat list of messages into chats grouped by chat_id."""
+        from collections import defaultdict
+
+        # Group messages by chat_id
+        grouped: dict[str, list[IngestMessage]] = defaultdict(list)
+
+        for raw_msg in messages:
+            if not isinstance(raw_msg, dict):
+                continue
+
+            chat_id = str(raw_msg.get("chat_id", raw_msg.get(self._chat_id_key, "default")))
+            msg = self._parse_message(raw_msg, chat_id, 0)
+            if msg:
+                grouped[chat_id].append(msg)
+
+        # Create ChatIngest for each group
+        chats: list[ChatIngest] = []
+        for chat_id, msgs in grouped.items():
+            msgs.sort(key=lambda m: m.timestamp)
+            min_timestamp = msgs[0].timestamp if msgs else 0
+
+            chats.append(
+                ChatIngest(
+                    source="generic_json",
+                    imported_chat_timestamp=min_timestamp,
+                    title=None,
+                    messages=msgs,
+                    conversation_id=chat_id,
+                )
+            )
+
+        return chats
+
+    def _parse_message(
+        self,
+        raw_msg: dict[str, Any],
+        chat_id: str,
+        default_timestamp: int,
+    ) -> IngestMessage | None:
+        """Parse a single message."""
+        # Get role
+        raw_role = raw_msg.get(self._role_key, "")
+        role = self._normalize_role(raw_role)
+        if not role:
+            return None
+
+        # Get content
+        if self._content_extractor:
+            content = self._content_extractor(raw_msg)
+        else:
+            content = raw_msg.get(self._content_key, "")
+            if isinstance(content, list):
+                content = "\n".join(str(c) for c in content)
+
+        if not content:
+            return None
+
+        # Get timestamp
+        timestamp = self._parse_timestamp(raw_msg.get(self._timestamp_key))
+        if timestamp == 0:
+            timestamp = default_timestamp
+
+        return IngestMessage(
+            role=role,  # type: ignore[arg-type]
+            content=str(content),
+            timestamp=timestamp,
+            chat_id=chat_id,
+        )
+
+    def _normalize_role(self, role: str) -> str | None:
+        """Normalize role to standard values."""
+        role_lower = str(role).lower()
+
+        # Check custom mapping first
+        if role_lower in self._role_mapping:
+            return self._role_mapping[role_lower]
+
+        # Standard mappings
+        if role_lower in ("user", "human"):
+            return "user"
+        if role_lower in ("assistant", "ai", "bot"):
+            return "assistant"
+        if role_lower == "system":
+            return "system"
+
+        return None
+
+    def _parse_timestamp(self, value: Any) -> int:
+        """Parse timestamp to epoch seconds."""
+        if value is None:
+            return 0
+        if isinstance(value, int):
+            return value
+        if isinstance(value, float):
+            return int(value)
+        if isinstance(value, str):
+            try:
+                # Try numeric string
+                return int(float(value))
+            except ValueError:
+                pass
+            try:
+                # Try ISO format
+                from datetime import datetime
+
+                dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+                return int(dt.timestamp())
+            except (ValueError, TypeError):
+                pass
+        return 0