basic-memory 0.17.1__py3-none-any.whl

basic_memory/importers/claude_conversations_importer.py

@@ -0,0 +1,180 @@
+"""Claude conversations import service for Basic Memory."""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List
+
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import ChatImportResult
+from basic_memory.importers.utils import clean_filename, format_timestamp
+
+logger = logging.getLogger(__name__)
+
+
+class ClaudeConversationsImporter(Importer[ChatImportResult]):
+    """Service for importing Claude conversations."""
+
+    async def import_data(
+        self, source_data, destination_folder: str, **kwargs: Any
+    ) -> ChatImportResult:
+        """Import conversations from Claude JSON export.
+
+        Args:
+            source_data: Path to the Claude conversations.json file.
+            destination_folder: Destination folder within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            ChatImportResult containing statistics and status of the import.
+        """
+        try:
+            # Ensure the destination folder exists
+            folder_path = self.ensure_folder_exists(destination_folder)
+
+            conversations = source_data
+
+            # Process each conversation
+            messages_imported = 0
+            chats_imported = 0
+
+            for chat in conversations:
+                # Get name, providing default for unnamed conversations
+                chat_name = chat.get("name") or f"Conversation {chat.get('uuid', 'untitled')}"
+
+                # Convert to entity
+                entity = self._format_chat_content(
+                    base_path=folder_path,
+                    name=chat_name,
+                    messages=chat["chat_messages"],
+                    created_at=chat["created_at"],
+                    modified_at=chat["updated_at"],
+                )
+
+                # Write file
+                file_path = self.base_path / Path(f"{entity.frontmatter.metadata['permalink']}.md")
+                await self.write_entity(entity, file_path)
+
+                chats_imported += 1
+                messages_imported += len(chat["chat_messages"])
+
+            return ChatImportResult(
+                import_count={"conversations": chats_imported, "messages": messages_imported},
+                success=True,
+                conversations=chats_imported,
+                messages=messages_imported,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import Claude conversations")
+            return self.handle_error("Failed to import Claude conversations", e)  # pyright: ignore [reportReturnType]
+
+    def _format_chat_content(
+        self,
+        base_path: Path,
+        name: str,
+        messages: List[Dict[str, Any]],
+        created_at: str,
+        modified_at: str,
+    ) -> EntityMarkdown:
+        """Convert chat messages to Basic Memory entity format.
+
+        Args:
+            base_path: Base path for the entity.
+            name: Chat name.
+            messages: List of chat messages.
+            created_at: Creation timestamp.
+            modified_at: Modification timestamp.
+
+        Returns:
+            EntityMarkdown instance representing the conversation.
+        """
+        # Generate permalink
+        date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
+        clean_title = clean_filename(name)
+        permalink = f"{base_path.name}/{date_prefix}-{clean_title}"
+
+        # Format content
+        content = self._format_chat_markdown(
+            name=name,
+            messages=messages,
+            created_at=created_at,
+            modified_at=modified_at,
+            permalink=permalink,
+        )
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "conversation",
+                    "title": name,
+                    "created": created_at,
+                    "modified": modified_at,
+                    "permalink": permalink,
+                }
+            ),
+            content=content,
+        )
+
+        return entity
+
+    def _format_chat_markdown(
+        self,
+        name: str,
+        messages: List[Dict[str, Any]],
+        created_at: str,
+        modified_at: str,
+        permalink: str,
+    ) -> str:
+        """Format chat as clean markdown.
+
+        Args:
+            name: Chat name.
+            messages: List of chat messages.
+            created_at: Creation timestamp.
+            modified_at: Modification timestamp.
+            permalink: Permalink for the entity.
+
+        Returns:
+            Formatted markdown content.
+        """
+        # Start with frontmatter and title
+        lines = [
+            f"# {name}\n",
+        ]
+
+        # Add messages
+        for msg in messages:
+            # Format timestamp
+            ts = format_timestamp(msg["created_at"])
+
+            # Add message header
+            lines.append(f"### {msg['sender'].title()} ({ts})")
+
+            # Handle message content
+            content = msg.get("text", "")
+            if msg.get("content"):
+                # Filter out None values before joining
+                content = " ".join(
+                    str(c.get("text", ""))
+                    for c in msg["content"]
+                    if c and c.get("text") is not None
+                )
+            lines.append(content)
+
+            # Handle attachments
+            attachments = msg.get("attachments", [])
+            for attachment in attachments:
+                if "file_name" in attachment:
+                    lines.append(f"\n**Attachment: {attachment['file_name']}**")
+                    if "extracted_content" in attachment:
+                        lines.append("```")
+                        lines.append(attachment["extracted_content"])
+                        lines.append("```")
+
+            # Add spacing between messages
+            lines.append("")
+
+        return "\n".join(lines)

basic_memory/importers/claude_projects_importer.py

@@ -0,0 +1,148 @@
+"""Claude projects import service for Basic Memory."""
+
+import logging
+from typing import Any, Dict, Optional
+
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import ProjectImportResult
+from basic_memory.importers.utils import clean_filename
+
+logger = logging.getLogger(__name__)
+
+
+class ClaudeProjectsImporter(Importer[ProjectImportResult]):
+    """Service for importing Claude projects."""
+
+    async def import_data(
+        self, source_data, destination_folder: str, **kwargs: Any
+    ) -> ProjectImportResult:
+        """Import projects from Claude JSON export.
+
+        Args:
+            source_path: Path to the Claude projects.json file.
+            destination_folder: Base folder for projects within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            ProjectImportResult containing statistics and status of the import.
+        """
+        try:
+            # Ensure the base folder exists
+            base_path = self.base_path
+            if destination_folder:
+                base_path = self.ensure_folder_exists(destination_folder)
+
+            projects = source_data
+
+            # Process each project
+            docs_imported = 0
+            prompts_imported = 0
+
+            for project in projects:
+                project_dir = clean_filename(project["name"])
+
+                # Create project directories
+                docs_dir = base_path / project_dir / "docs"
+                docs_dir.mkdir(parents=True, exist_ok=True)
+
+                # Import prompt template if it exists
+                if prompt_entity := self._format_prompt_markdown(project):
+                    file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
+                    await self.write_entity(prompt_entity, file_path)
+                    prompts_imported += 1
+
+                # Import project documents
+                for doc in project.get("docs", []):
+                    entity = self._format_project_markdown(project, doc)
+                    file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
+                    await self.write_entity(entity, file_path)
+                    docs_imported += 1
+
+            return ProjectImportResult(
+                import_count={"documents": docs_imported, "prompts": prompts_imported},
+                success=True,
+                documents=docs_imported,
+                prompts=prompts_imported,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import Claude projects")
+            return self.handle_error("Failed to import Claude projects", e)  # pyright: ignore [reportReturnType]
+
+    def _format_project_markdown(
+        self, project: Dict[str, Any], doc: Dict[str, Any]
+    ) -> EntityMarkdown:
+        """Format a project document as a Basic Memory entity.
+
+        Args:
+            project: Project data.
+            doc: Document data.
+
+        Returns:
+            EntityMarkdown instance representing the document.
+        """
+        # Extract timestamps
+        created_at = doc.get("created_at") or project["created_at"]
+        modified_at = project["updated_at"]
+
+        # Generate clean names for organization
+        project_dir = clean_filename(project["name"])
+        doc_file = clean_filename(doc["filename"])
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "project_doc",
+                    "title": doc["filename"],
+                    "created": created_at,
+                    "modified": modified_at,
+                    "permalink": f"{project_dir}/docs/{doc_file}",
+                    "project_name": project["name"],
+                    "project_uuid": project["uuid"],
+                    "doc_uuid": doc["uuid"],
+                }
+            ),
+            content=doc["content"],
+        )
+
+        return entity
+
+    def _format_prompt_markdown(self, project: Dict[str, Any]) -> Optional[EntityMarkdown]:
+        """Format project prompt template as a Basic Memory entity.
+
+        Args:
+            project: Project data.
+
+        Returns:
+            EntityMarkdown instance representing the prompt template, or None if
+            no prompt template exists.
+        """
+        if not project.get("prompt_template"):
+            return None
+
+        # Extract timestamps
+        created_at = project["created_at"]
+        modified_at = project["updated_at"]
+
+        # Generate clean project directory name
+        project_dir = clean_filename(project["name"])
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "prompt_template",
+                    "title": f"Prompt Template: {project['name']}",
+                    "created": created_at,
+                    "modified": modified_at,
+                    "permalink": f"{project_dir}/prompt-template",
+                    "project_name": project["name"],
+                    "project_uuid": project["uuid"],
+                }
+            ),
+            content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}",
+        )
+
+        return entity

basic_memory/importers/memory_json_importer.py

@@ -0,0 +1,108 @@
+"""Memory JSON import service for Basic Memory."""
+
+import logging
+from typing import Any, Dict, List
+
+from basic_memory.config import get_project_config
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown, Observation, Relation
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import EntityImportResult
+
+logger = logging.getLogger(__name__)
+
+
+class MemoryJsonImporter(Importer[EntityImportResult]):
+    """Service for importing memory.json format data."""
+
+    async def import_data(
+        self, source_data, destination_folder: str = "", **kwargs: Any
+    ) -> EntityImportResult:
+        """Import entities and relations from a memory.json file.
+
+        Args:
+            source_data: Path to the memory.json file.
+            destination_folder: Optional destination folder within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            EntityImportResult containing statistics and status of the import.
+        """
+        config = get_project_config()
+        try:
+            # First pass - collect all relations by source entity
+            entity_relations: Dict[str, List[Relation]] = {}
+            entities: Dict[str, Dict[str, Any]] = {}
+            skipped_entities: int = 0
+
+            # Ensure the base path exists
+            base_path = config.home  # pragma: no cover
+            if destination_folder:  # pragma: no cover
+                base_path = self.ensure_folder_exists(destination_folder)
+
+            # First pass - collect entities and relations
+            for line in source_data:
+                data = line
+                if data["type"] == "entity":
+                    # Handle different possible name keys
+                    entity_name = data.get("name") or data.get("entityName") or data.get("id")
+                    if not entity_name:
+                        logger.warning(f"Entity missing name field: {data}")
+                        skipped_entities += 1
+                        continue
+                    entities[entity_name] = data
+                elif data["type"] == "relation":
+                    # Store relation with its source entity
+                    source = data.get("from") or data.get("from_id")
+                    if source not in entity_relations:
+                        entity_relations[source] = []
+                    entity_relations[source].append(
+                        Relation(
+                            type=data.get("relationType") or data.get("relation_type"),
+                            target=data.get("to") or data.get("to_id"),
+                        )
+                    )
+
+            # Second pass - create and write entities
+            entities_created = 0
+            for name, entity_data in entities.items():
+                # Get entity type with fallback
+                entity_type = entity_data.get("entityType") or entity_data.get("type") or "entity"
+
+                # Ensure entity type directory exists
+                entity_type_dir = base_path / entity_type
+                entity_type_dir.mkdir(parents=True, exist_ok=True)
+
+                # Get observations with fallback to empty list
+                observations = entity_data.get("observations", [])
+
+                entity = EntityMarkdown(
+                    frontmatter=EntityFrontmatter(
+                        metadata={
+                            "type": entity_type,
+                            "title": name,
+                            "permalink": f"{entity_type}/{name}",
+                        }
+                    ),
+                    content=f"# {name}\n",
+                    observations=[Observation(content=obs) for obs in observations],
+                    relations=entity_relations.get(name, []),
+                )
+
+                # Write entity file
+                file_path = base_path / f"{entity_type}/{name}.md"
+                await self.write_entity(entity, file_path)
+                entities_created += 1
+
+            relations_count = sum(len(rels) for rels in entity_relations.values())
+
+            return EntityImportResult(
+                import_count={"entities": entities_created, "relations": relations_count},
+                success=True,
+                entities=entities_created,
+                relations=relations_count,
+                skipped_entities=skipped_entities,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import memory.json")
+            return self.handle_error("Failed to import memory.json", e)  # pyright: ignore [reportReturnType]

basic_memory/importers/utils.py

@@ -0,0 +1,61 @@
+"""Utility functions for import services."""
+
+import re
+from datetime import datetime
+from typing import Any
+
+
+def clean_filename(name: str | None) -> str:  # pragma: no cover
+    """Clean a string to be used as a filename.
+
+    Args:
+        name: The string to clean (can be None).
+
+    Returns:
+        A cleaned string suitable for use as a filename.
+    """
+    # Handle None or empty input
+    if not name:
+        return "untitled"
+    # Replace common punctuation and whitespace with underscores
+    name = re.sub(r"[\s\-,.:/\\\[\]\(\)]+", "_", name)
+    # Remove any non-alphanumeric or underscore characters
+    name = re.sub(r"[^\w]+", "", name)
+    # Ensure the name isn't too long
+    if len(name) > 100:  # pragma: no cover
+        name = name[:100]
+    # Ensure the name isn't empty
+    if not name:  # pragma: no cover
+        name = "untitled"
+    return name
+
+
+def format_timestamp(timestamp: Any) -> str:  # pragma: no cover
+    """Format a timestamp for use in a filename or title.
+
+    Args:
+        timestamp: A timestamp in various formats.
+
+    Returns:
+        A formatted string representation of the timestamp.
+    """
+    if isinstance(timestamp, str):
+        try:
+            # Try ISO format
+            timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+        except ValueError:
+            try:
+                # Try unix timestamp as string
+                timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
+            except ValueError:
+                # Return as is if we can't parse it
+                return timestamp
+    elif isinstance(timestamp, (int, float)):
+        # Unix timestamp
+        timestamp = datetime.fromtimestamp(timestamp).astimezone()
+
+    if isinstance(timestamp, datetime):
+        return timestamp.strftime("%Y-%m-%d %H:%M:%S")
+
+    # Return as is if we can't format it
+    return str(timestamp)  # pragma: no cover

basic_memory/markdown/__init__.py

@@ -0,0 +1,21 @@
+"""Base package for markdown parsing."""
+
+from basic_memory.file_utils import ParseError
+from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
+from basic_memory.markdown.schemas import (
+    EntityMarkdown,
+    EntityFrontmatter,
+    Observation,
+    Relation,
+)
+
+__all__ = [
+    "EntityMarkdown",
+    "EntityFrontmatter",
+    "EntityParser",
+    "MarkdownProcessor",
+    "Observation",
+    "Relation",
+    "ParseError",
+]