basic-memory 0.2.12__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/importers/chatgpt_importer.py

@@ -0,0 +1,232 @@
+"""ChatGPT import service for Basic Memory."""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Set
+
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import ChatImportResult
+from basic_memory.importers.utils import clean_filename, format_timestamp
+
+logger = logging.getLogger(__name__)
+
+
+class ChatGPTImporter(Importer[ChatImportResult]):
+    """Service for importing ChatGPT conversations."""
+
+    async def import_data(
+        self, source_data, destination_folder: str, **kwargs: Any
+    ) -> ChatImportResult:
+        """Import conversations from ChatGPT JSON export.
+
+        Args:
+            source_path: Path to the ChatGPT conversations.json file.
+            destination_folder: Destination folder within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            ChatImportResult containing statistics and status of the import.
+        """
+        try:  # pragma: no cover
+            # Ensure the destination folder exists
+            self.ensure_folder_exists(destination_folder)
+            conversations = source_data
+
+            # Process each conversation
+            messages_imported = 0
+            chats_imported = 0
+
+            for chat in conversations:
+                # Convert to entity
+                entity = self._format_chat_content(destination_folder, chat)
+
+                # Write file
+                file_path = self.base_path / f"{entity.frontmatter.metadata['permalink']}.md"
+                await self.write_entity(entity, file_path)
+
+                # Count messages
+                msg_count = sum(
+                    1
+                    for node in chat["mapping"].values()
+                    if node.get("message")
+                    and not node.get("message", {})
+                    .get("metadata", {})
+                    .get("is_visually_hidden_from_conversation")
+                )
+
+                chats_imported += 1
+                messages_imported += msg_count
+
+            return ChatImportResult(
+                import_count={"conversations": chats_imported, "messages": messages_imported},
+                success=True,
+                conversations=chats_imported,
+                messages=messages_imported,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import ChatGPT conversations")
+            return self.handle_error("Failed to import ChatGPT conversations", e)  # pyright: ignore [reportReturnType]
+
+    def _format_chat_content(
+        self, folder: str, conversation: Dict[str, Any]
+    ) -> EntityMarkdown:  # pragma: no cover
+        """Convert chat conversation to Basic Memory entity.
+
+        Args:
+            folder: Destination folder name.
+            conversation: ChatGPT conversation data.
+
+        Returns:
+            EntityMarkdown instance representing the conversation.
+        """
+        # Extract timestamps
+        created_at = conversation["create_time"]
+        modified_at = conversation["update_time"]
+
+        root_id = None
+        # Find root message
+        for node_id, node in conversation["mapping"].items():
+            if node.get("parent") is None:
+                root_id = node_id
+                break
+
+        # Generate permalink
+        date_prefix = datetime.fromtimestamp(created_at).astimezone().strftime("%Y%m%d")
+        clean_title = clean_filename(conversation["title"])
+
+        # Format content
+        content = self._format_chat_markdown(
+            title=conversation["title"],
+            mapping=conversation["mapping"],
+            root_id=root_id,
+            created_at=created_at,
+            modified_at=modified_at,
+        )
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "conversation",
+                    "title": conversation["title"],
+                    "created": format_timestamp(created_at),
+                    "modified": format_timestamp(modified_at),
+                    "permalink": f"{folder}/{date_prefix}-{clean_title}",
+                }
+            ),
+            content=content,
+        )
+
+        return entity
+
+    def _format_chat_markdown(
+        self,
+        title: str,
+        mapping: Dict[str, Any],
+        root_id: Optional[str],
+        created_at: float,
+        modified_at: float,
+    ) -> str:  # pragma: no cover
+        """Format chat as clean markdown.
+
+        Args:
+            title: Chat title.
+            mapping: Message mapping.
+            root_id: Root message ID.
+            created_at: Creation timestamp.
+            modified_at: Modification timestamp.
+
+        Returns:
+            Formatted markdown content.
+        """
+        # Start with title
+        lines = [f"# {title}\n"]
+
+        # Traverse message tree
+        seen_msgs: Set[str] = set()
+        messages = self._traverse_messages(mapping, root_id, seen_msgs)
+
+        # Format each message
+        for msg in messages:
+            # Skip hidden messages
+            if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
+                continue
+
+            # Get author and timestamp
+            author = msg["author"]["role"].title()
+            ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
+
+            # Add message header
+            lines.append(f"### {author} ({ts})")
+
+            # Add message content
+            content = self._get_message_content(msg)
+            if content:
+                lines.append(content)
+
+            # Add spacing
+            lines.append("")
+
+        return "\n".join(lines)
+
+    def _get_message_content(self, message: Dict[str, Any]) -> str:  # pragma: no cover
+        """Extract clean message content.
+
+        Args:
+            message: Message data.
+
+        Returns:
+            Cleaned message content.
+        """
+        if not message or "content" not in message:
+            return ""
+
+        content = message["content"]
+        if content.get("content_type") == "text":
+            return "\n".join(content.get("parts", []))
+        elif content.get("content_type") == "code":
+            return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
+        return ""
+
+    def _traverse_messages(
+        self, mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
+    ) -> List[Dict[str, Any]]:  # pragma: no cover
+        """Traverse message tree iteratively to handle deep conversations.
+
+        Args:
+            mapping: Message mapping.
+            root_id: Root message ID.
+            seen: Set of seen message IDs.
+
+        Returns:
+            List of message data.
+        """
+        messages = []
+        if not root_id:
+            return messages
+
+        # Use iterative approach with stack to avoid recursion depth issues
+        stack = [root_id]
+
+        while stack:
+            node_id = stack.pop()
+            if not node_id:
+                continue
+
+            node = mapping.get(node_id)
+            if not node:
+                continue
+
+            # Process current node if it has a message and hasn't been seen
+            if node["id"] not in seen and node.get("message"):
+                seen.add(node["id"])
+                messages.append(node["message"])
+
+            # Add children to stack in reverse order to maintain conversation flow
+            children = node.get("children", [])
+            for child_id in reversed(children):
+                stack.append(child_id)
+
+        return messages

basic_memory/importers/claude_conversations_importer.py

@@ -0,0 +1,177 @@
+"""Claude conversations import service for Basic Memory."""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List
+
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import ChatImportResult
+from basic_memory.importers.utils import clean_filename, format_timestamp
+
+logger = logging.getLogger(__name__)
+
+
+class ClaudeConversationsImporter(Importer[ChatImportResult]):
+    """Service for importing Claude conversations."""
+
+    async def import_data(
+        self, source_data, destination_folder: str, **kwargs: Any
+    ) -> ChatImportResult:
+        """Import conversations from Claude JSON export.
+
+        Args:
+            source_data: Path to the Claude conversations.json file.
+            destination_folder: Destination folder within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            ChatImportResult containing statistics and status of the import.
+        """
+        try:
+            # Ensure the destination folder exists
+            folder_path = self.ensure_folder_exists(destination_folder)
+
+            conversations = source_data
+
+            # Process each conversation
+            messages_imported = 0
+            chats_imported = 0
+
+            for chat in conversations:
+                # Convert to entity
+                entity = self._format_chat_content(
+                    base_path=folder_path,
+                    name=chat["name"],
+                    messages=chat["chat_messages"],
+                    created_at=chat["created_at"],
+                    modified_at=chat["updated_at"],
+                )
+
+                # Write file
+                file_path = self.base_path / Path(f"{entity.frontmatter.metadata['permalink']}.md")
+                await self.write_entity(entity, file_path)
+
+                chats_imported += 1
+                messages_imported += len(chat["chat_messages"])
+
+            return ChatImportResult(
+                import_count={"conversations": chats_imported, "messages": messages_imported},
+                success=True,
+                conversations=chats_imported,
+                messages=messages_imported,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import Claude conversations")
+            return self.handle_error("Failed to import Claude conversations", e)  # pyright: ignore [reportReturnType]
+
+    def _format_chat_content(
+        self,
+        base_path: Path,
+        name: str,
+        messages: List[Dict[str, Any]],
+        created_at: str,
+        modified_at: str,
+    ) -> EntityMarkdown:
+        """Convert chat messages to Basic Memory entity format.
+
+        Args:
+            base_path: Base path for the entity.
+            name: Chat name.
+            messages: List of chat messages.
+            created_at: Creation timestamp.
+            modified_at: Modification timestamp.
+
+        Returns:
+            EntityMarkdown instance representing the conversation.
+        """
+        # Generate permalink
+        date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
+        clean_title = clean_filename(name)
+        permalink = f"{base_path.name}/{date_prefix}-{clean_title}"
+
+        # Format content
+        content = self._format_chat_markdown(
+            name=name,
+            messages=messages,
+            created_at=created_at,
+            modified_at=modified_at,
+            permalink=permalink,
+        )
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "conversation",
+                    "title": name,
+                    "created": created_at,
+                    "modified": modified_at,
+                    "permalink": permalink,
+                }
+            ),
+            content=content,
+        )
+
+        return entity
+
+    def _format_chat_markdown(
+        self,
+        name: str,
+        messages: List[Dict[str, Any]],
+        created_at: str,
+        modified_at: str,
+        permalink: str,
+    ) -> str:
+        """Format chat as clean markdown.
+
+        Args:
+            name: Chat name.
+            messages: List of chat messages.
+            created_at: Creation timestamp.
+            modified_at: Modification timestamp.
+            permalink: Permalink for the entity.
+
+        Returns:
+            Formatted markdown content.
+        """
+        # Start with frontmatter and title
+        lines = [
+            f"# {name}\n",
+        ]
+
+        # Add messages
+        for msg in messages:
+            # Format timestamp
+            ts = format_timestamp(msg["created_at"])
+
+            # Add message header
+            lines.append(f"### {msg['sender'].title()} ({ts})")
+
+            # Handle message content
+            content = msg.get("text", "")
+            if msg.get("content"):
+                # Filter out None values before joining
+                content = " ".join(
+                    str(c.get("text", ""))
+                    for c in msg["content"]
+                    if c and c.get("text") is not None
+                )
+            lines.append(content)
+
+            # Handle attachments
+            attachments = msg.get("attachments", [])
+            for attachment in attachments:
+                if "file_name" in attachment:
+                    lines.append(f"\n**Attachment: {attachment['file_name']}**")
+                    if "extracted_content" in attachment:
+                        lines.append("```")
+                        lines.append(attachment["extracted_content"])
+                        lines.append("```")
+
+            # Add spacing between messages
+            lines.append("")
+
+        return "\n".join(lines)

basic_memory/importers/claude_projects_importer.py

@@ -0,0 +1,148 @@
+"""Claude projects import service for Basic Memory."""
+
+import logging
+from typing import Any, Dict, Optional
+
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import ProjectImportResult
+from basic_memory.importers.utils import clean_filename
+
+logger = logging.getLogger(__name__)
+
+
+class ClaudeProjectsImporter(Importer[ProjectImportResult]):
+    """Service for importing Claude projects."""
+
+    async def import_data(
+        self, source_data, destination_folder: str, **kwargs: Any
+    ) -> ProjectImportResult:
+        """Import projects from Claude JSON export.
+
+        Args:
+            source_path: Path to the Claude projects.json file.
+            destination_folder: Base folder for projects within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            ProjectImportResult containing statistics and status of the import.
+        """
+        try:
+            # Ensure the base folder exists
+            base_path = self.base_path
+            if destination_folder:
+                base_path = self.ensure_folder_exists(destination_folder)
+
+            projects = source_data
+
+            # Process each project
+            docs_imported = 0
+            prompts_imported = 0
+
+            for project in projects:
+                project_dir = clean_filename(project["name"])
+
+                # Create project directories
+                docs_dir = base_path / project_dir / "docs"
+                docs_dir.mkdir(parents=True, exist_ok=True)
+
+                # Import prompt template if it exists
+                if prompt_entity := self._format_prompt_markdown(project):
+                    file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
+                    await self.write_entity(prompt_entity, file_path)
+                    prompts_imported += 1
+
+                # Import project documents
+                for doc in project.get("docs", []):
+                    entity = self._format_project_markdown(project, doc)
+                    file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
+                    await self.write_entity(entity, file_path)
+                    docs_imported += 1
+
+            return ProjectImportResult(
+                import_count={"documents": docs_imported, "prompts": prompts_imported},
+                success=True,
+                documents=docs_imported,
+                prompts=prompts_imported,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import Claude projects")
+            return self.handle_error("Failed to import Claude projects", e)  # pyright: ignore [reportReturnType]
+
+    def _format_project_markdown(
+        self, project: Dict[str, Any], doc: Dict[str, Any]
+    ) -> EntityMarkdown:
+        """Format a project document as a Basic Memory entity.
+
+        Args:
+            project: Project data.
+            doc: Document data.
+
+        Returns:
+            EntityMarkdown instance representing the document.
+        """
+        # Extract timestamps
+        created_at = doc.get("created_at") or project["created_at"]
+        modified_at = project["updated_at"]
+
+        # Generate clean names for organization
+        project_dir = clean_filename(project["name"])
+        doc_file = clean_filename(doc["filename"])
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "project_doc",
+                    "title": doc["filename"],
+                    "created": created_at,
+                    "modified": modified_at,
+                    "permalink": f"{project_dir}/docs/{doc_file}",
+                    "project_name": project["name"],
+                    "project_uuid": project["uuid"],
+                    "doc_uuid": doc["uuid"],
+                }
+            ),
+            content=doc["content"],
+        )
+
+        return entity
+
+    def _format_prompt_markdown(self, project: Dict[str, Any]) -> Optional[EntityMarkdown]:
+        """Format project prompt template as a Basic Memory entity.
+
+        Args:
+            project: Project data.
+
+        Returns:
+            EntityMarkdown instance representing the prompt template, or None if
+            no prompt template exists.
+        """
+        if not project.get("prompt_template"):
+            return None
+
+        # Extract timestamps
+        created_at = project["created_at"]
+        modified_at = project["updated_at"]
+
+        # Generate clean project directory name
+        project_dir = clean_filename(project["name"])
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "prompt_template",
+                    "title": f"Prompt Template: {project['name']}",
+                    "created": created_at,
+                    "modified": modified_at,
+                    "permalink": f"{project_dir}/prompt-template",
+                    "project_name": project["name"],
+                    "project_uuid": project["uuid"],
+                }
+            ),
+            content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}",
+        )
+
+        return entity

basic_memory/importers/memory_json_importer.py

@@ -0,0 +1,108 @@
+"""Memory JSON import service for Basic Memory."""
+
+import logging
+from typing import Any, Dict, List
+
+from basic_memory.config import get_project_config
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown, Observation, Relation
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import EntityImportResult
+
+logger = logging.getLogger(__name__)
+
+
+class MemoryJsonImporter(Importer[EntityImportResult]):
+    """Service for importing memory.json format data."""
+
+    async def import_data(
+        self, source_data, destination_folder: str = "", **kwargs: Any
+    ) -> EntityImportResult:
+        """Import entities and relations from a memory.json file.
+
+        Args:
+            source_data: Path to the memory.json file.
+            destination_folder: Optional destination folder within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            EntityImportResult containing statistics and status of the import.
+        """
+        config = get_project_config()
+        try:
+            # First pass - collect all relations by source entity
+            entity_relations: Dict[str, List[Relation]] = {}
+            entities: Dict[str, Dict[str, Any]] = {}
+            skipped_entities: int = 0
+
+            # Ensure the base path exists
+            base_path = config.home  # pragma: no cover
+            if destination_folder:  # pragma: no cover
+                base_path = self.ensure_folder_exists(destination_folder)
+
+            # First pass - collect entities and relations
+            for line in source_data:
+                data = line
+                if data["type"] == "entity":
+                    # Handle different possible name keys
+                    entity_name = data.get("name") or data.get("entityName") or data.get("id")
+                    if not entity_name:
+                        logger.warning(f"Entity missing name field: {data}")
+                        skipped_entities += 1
+                        continue
+                    entities[entity_name] = data
+                elif data["type"] == "relation":
+                    # Store relation with its source entity
+                    source = data.get("from") or data.get("from_id")
+                    if source not in entity_relations:
+                        entity_relations[source] = []
+                    entity_relations[source].append(
+                        Relation(
+                            type=data.get("relationType") or data.get("relation_type"),
+                            target=data.get("to") or data.get("to_id"),
+                        )
+                    )
+
+            # Second pass - create and write entities
+            entities_created = 0
+            for name, entity_data in entities.items():
+                # Get entity type with fallback
+                entity_type = entity_data.get("entityType") or entity_data.get("type") or "entity"
+
+                # Ensure entity type directory exists
+                entity_type_dir = base_path / entity_type
+                entity_type_dir.mkdir(parents=True, exist_ok=True)
+
+                # Get observations with fallback to empty list
+                observations = entity_data.get("observations", [])
+
+                entity = EntityMarkdown(
+                    frontmatter=EntityFrontmatter(
+                        metadata={
+                            "type": entity_type,
+                            "title": name,
+                            "permalink": f"{entity_type}/{name}",
+                        }
+                    ),
+                    content=f"# {name}\n",
+                    observations=[Observation(content=obs) for obs in observations],
+                    relations=entity_relations.get(name, []),
+                )
+
+                # Write entity file
+                file_path = base_path / f"{entity_type}/{name}.md"
+                await self.write_entity(entity, file_path)
+                entities_created += 1
+
+            relations_count = sum(len(rels) for rels in entity_relations.values())
+
+            return EntityImportResult(
+                import_count={"entities": entities_created, "relations": relations_count},
+                success=True,
+                entities=entities_created,
+                relations=relations_count,
+                skipped_entities=skipped_entities,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import memory.json")
+            return self.handle_error("Failed to import memory.json", e)  # pyright: ignore [reportReturnType]