basic-memory 0.12.3__py3-none-any.whl → 0.13.0b2__py3-none-any.whl

basic_memory/cli/commands/import_chatgpt.py

@@ -2,203 +2,21 @@
 
 import asyncio
 import json
-from datetime import datetime
 from pathlib import Path
-from typing import Dict, Any, List, Annotated, Set, Optional
+from typing import Annotated
 
 import typer
 from basic_memory.cli.app import import_app
 from basic_memory.config import config
+from basic_memory.importers import ChatGPTImporter
 from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_timestamp(ts: float) -> str:
-    """Format Unix timestamp for display."""
-    dt = datetime.fromtimestamp(ts)
-    return dt.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def get_message_content(message: Dict[str, Any]) -> str:
-    """Extract clean message content."""
-    if not message or "content" not in message:
-        return ""  # pragma: no cover
-
-    content = message["content"]
-    if content.get("content_type") == "text":
-        return "\n".join(content.get("parts", []))
-    elif content.get("content_type") == "code":
-        return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
-    return ""  # pragma: no cover
-
-
-def traverse_messages(
-    mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
-) -> List[Dict[str, Any]]:
-    """Traverse message tree and return messages in order."""
-    messages = []
-    node = mapping.get(root_id) if root_id else None
-
-    while node:
-        if node["id"] not in seen and node.get("message"):
-            seen.add(node["id"])
-            messages.append(node["message"])
-
-        # Follow children
-        children = node.get("children", [])
-        for child_id in children:
-            child_msgs = traverse_messages(mapping, child_id, seen)
-            messages.extend(child_msgs)
-
-        break  # Don't follow siblings
-
-    return messages
-
-
-def format_chat_markdown(
-    title: str,
-    mapping: Dict[str, Any],
-    root_id: Optional[str],
-    created_at: float,
-    modified_at: float,
-) -> str:
-    """Format chat as clean markdown."""
-
-    # Start with title
-    lines = [f"# {title}\n"]
-
-    # Traverse message tree
-    seen_msgs = set()
-    messages = traverse_messages(mapping, root_id, seen_msgs)
-
-    # Format each message
-    for msg in messages:
-        # Skip hidden messages
-        if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
-            continue
-
-        # Get author and timestamp
-        author = msg["author"]["role"].title()
-        ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
-
-        # Add message header
-        lines.append(f"### {author} ({ts})")
-
-        # Add message content
-        content = get_message_content(msg)
-        if content:
-            lines.append(content)
-
-        # Add spacing
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def format_chat_content(folder: str, conversation: Dict[str, Any]) -> EntityMarkdown:
-    """Convert chat conversation to Basic Memory entity."""
-
-    # Extract timestamps
-    created_at = conversation["create_time"]
-    modified_at = conversation["update_time"]
-
-    root_id = None
-    # Find root message
-    for node_id, node in conversation["mapping"].items():
-        if node.get("parent") is None:
-            root_id = node_id
-            break
-
-    # Generate permalink
-    date_prefix = datetime.fromtimestamp(created_at).strftime("%Y%m%d")
-    clean_title = clean_filename(conversation["title"])
-
-    # Format content
-    content = format_chat_markdown(
-        title=conversation["title"],
-        mapping=conversation["mapping"],
-        root_id=root_id,
-        created_at=created_at,
-        modified_at=modified_at,
-    )
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "conversation",
-                "title": conversation["title"],
-                "created": format_timestamp(created_at),
-                "modified": format_timestamp(modified_at),
-                "permalink": f"{folder}/{date_prefix}-{clean_title}",
-            }
-        ),
-        content=content,
-    )
-
-    return entity
-
-
-async def process_chatgpt_json(
-    json_path: Path, folder: str, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import conversations from ChatGPT JSON format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading chat data...", total=None)
-
-        # Read conversations
-        conversations = json.loads(json_path.read_text(encoding="utf-8"))
-        progress.update(read_task, total=len(conversations))
-
-        # Process each conversation
-        messages_imported = 0
-        chats_imported = 0
-
-        for chat in conversations:
-            # Convert to entity
-            entity = format_chat_content(folder, chat)
-
-            # Write file
-            file_path = config.home / f"{entity.frontmatter.metadata['permalink']}.md"
-            # logger.info(f"Writing file: {file_path.absolute()}")
-            await markdown_processor.write_file(file_path, entity)
-
-            # Count messages
-            msg_count = sum(
-                1
-                for node in chat["mapping"].values()
-                if node.get("message")
-                and not node.get("message", {})
-                .get("metadata", {})
-                .get("is_visually_hidden_from_conversation")
-            )
-
-            chats_imported += 1
-            messages_imported += msg_count
-            progress.update(read_task, advance=1)
-
-    return {"conversations": chats_imported, "messages": messages_imported}
-
-
 async def get_markdown_processor() -> MarkdownProcessor:
     """Get MarkdownProcessor instance."""
     entity_parser = EntityParser(config.home)
@@ -225,30 +43,36 @@ def import_chatgpt(
     """
 
     try:
-        if conversations_json:
-            if not conversations_json.exists():
-                typer.echo(f"Error: File not found: {conversations_json}", err=True)
-                raise typer.Exit(1)
-
-            # Get markdown processor
-            markdown_processor = asyncio.run(get_markdown_processor())
-
-            # Process the file
-            base_path = config.home / folder
-            console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
-            results = asyncio.run(
-                process_chatgpt_json(conversations_json, folder, markdown_processor)
-            )
-
-            # Show results
-            console.print(
-                Panel(
-                    f"[green]Import complete![/green]\n\n"
-                    f"Imported {results['conversations']} conversations\n"
-                    f"Containing {results['messages']} messages",
-                    expand=False,
-                )
+        if not conversations_json.exists():  # pragma: no cover
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
+
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
+
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
+
+        # Create importer and run import
+        importer = ChatGPTImporter(config.home, markdown_processor)
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
+
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
             )
+        )
 
         console.print("\nRun 'basic-memory sync' to index the new files.")
 

basic_memory/cli/commands/import_claude_conversations.py

@@ -2,156 +2,21 @@
 
 import asyncio
 import json
-from datetime import datetime
 from pathlib import Path
-from typing import Dict, Any, List, Annotated
+from typing import Annotated
 
 import typer
 from basic_memory.cli.app import claude_app
 from basic_memory.config import config
+from basic_memory.importers.claude_conversations_importer import ClaudeConversationsImporter
 from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    # Remove invalid characters and convert spaces
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_timestamp(ts: str) -> str:
-    """Format ISO timestamp for display."""
-    dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
-    return dt.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def format_chat_markdown(
-    name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str
-) -> str:
-    """Format chat as clean markdown."""
-
-    # Start with frontmatter and title
-    lines = [
-        f"# {name}\n",
-    ]
-
-    # Add messages
-    for msg in messages:
-        # Format timestamp
-        ts = format_timestamp(msg["created_at"])
-
-        # Add message header
-        lines.append(f"### {msg['sender'].title()} ({ts})")
-
-        # Handle message content
-        content = msg.get("text", "")
-        if msg.get("content"):
-            content = " ".join(c.get("text", "") for c in msg["content"])
-        lines.append(content)
-
-        # Handle attachments
-        attachments = msg.get("attachments", [])
-        for attachment in attachments:
-            if "file_name" in attachment:
-                lines.append(f"\n**Attachment: {attachment['file_name']}**")
-                if "extracted_content" in attachment:
-                    lines.append("```")
-                    lines.append(attachment["extracted_content"])
-                    lines.append("```")
-
-        # Add spacing between messages
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def format_chat_content(
-    base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str
-) -> EntityMarkdown:
-    """Convert chat messages to Basic Memory entity format."""
-
-    # Generate permalink
-    date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
-    clean_title = clean_filename(name)
-    permalink = f"{base_path}/{date_prefix}-{clean_title}"
-
-    # Format content
-    content = format_chat_markdown(
-        name=name,
-        messages=messages,
-        created_at=created_at,
-        modified_at=modified_at,
-        permalink=permalink,
-    )
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "conversation",
-                "title": name,
-                "created": created_at,
-                "modified": modified_at,
-                "permalink": permalink,
-            }
-        ),
-        content=content,
-    )
-
-    return entity
-
-
-async def process_conversations_json(
-    json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import chat data from conversations2.json format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading chat data...", total=None)
-
-        # Read chat data - handle array of arrays format
-        data = json.loads(json_path.read_text(encoding="utf-8"))
-        conversations = [chat for chat in data]
-        progress.update(read_task, total=len(conversations))
-
-        # Process each conversation
-        messages_imported = 0
-        chats_imported = 0
-
-        for chat in conversations:
-            # Convert to entity
-            entity = format_chat_content(
-                base_path=base_path,
-                name=chat["name"],
-                messages=chat["chat_messages"],
-                created_at=chat["created_at"],
-                modified_at=chat["updated_at"],
-            )
-
-            # Write file
-            file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md")
-            await markdown_processor.write_file(file_path, entity)
-
-            chats_imported += 1
-            messages_imported += len(chat["chat_messages"])
-            progress.update(read_task, advance=1)
-
-    return {"conversations": chats_imported, "messages": messages_imported}
-
-
 async def get_markdown_processor() -> MarkdownProcessor:
     """Get MarkdownProcessor instance."""
     entity_parser = EntityParser(config.home)
@@ -185,19 +50,28 @@ def import_claude(
         # Get markdown processor
         markdown_processor = asyncio.run(get_markdown_processor())
 
+        # Create the importer
+        importer = ClaudeConversationsImporter(config.home, markdown_processor)
+
         # Process the file
         base_path = config.home / folder
         console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
-        results = asyncio.run(
-            process_conversations_json(conversations_json, base_path, markdown_processor)
-        )
+
+        # Run the import
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
 
         # Show results
         console.print(
             Panel(
                 f"[green]Import complete![/green]\n\n"
-                f"Imported {results['conversations']} conversations\n"
-                f"Containing {results['messages']} messages",
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
                 expand=False,
             )
         )

basic_memory/cli/commands/import_claude_projects.py

@@ -3,138 +3,20 @@
 import asyncio
 import json
 from pathlib import Path
-from typing import Dict, Any, Annotated, Optional
+from typing import Annotated
 
 import typer
 from basic_memory.cli.app import claude_app
 from basic_memory.config import config
+from basic_memory.importers.claude_projects_importer import ClaudeProjectsImporter
 from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_project_markdown(project: Dict[str, Any], doc: Dict[str, Any]) -> EntityMarkdown:
-    """Format a project document as a Basic Memory entity."""
-
-    # Extract timestamps
-    created_at = doc.get("created_at") or project["created_at"]
-    modified_at = project["updated_at"]
-
-    # Generate clean names for organization
-    project_dir = clean_filename(project["name"])
-    doc_file = clean_filename(doc["filename"])
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "project_doc",
-                "title": doc["filename"],
-                "created": created_at,
-                "modified": modified_at,
-                "permalink": f"{project_dir}/docs/{doc_file}",
-                "project_name": project["name"],
-                "project_uuid": project["uuid"],
-                "doc_uuid": doc["uuid"],
-            }
-        ),
-        content=doc["content"],
-    )
-
-    return entity
-
-
-def format_prompt_markdown(project: Dict[str, Any]) -> Optional[EntityMarkdown]:
-    """Format project prompt template as a Basic Memory entity."""
-
-    if not project.get("prompt_template"):
-        return None
-
-    # Extract timestamps
-    created_at = project["created_at"]
-    modified_at = project["updated_at"]
-
-    # Generate clean project directory name
-    project_dir = clean_filename(project["name"])
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "prompt_template",
-                "title": f"Prompt Template: {project['name']}",
-                "created": created_at,
-                "modified": modified_at,
-                "permalink": f"{project_dir}/prompt-template",
-                "project_name": project["name"],
-                "project_uuid": project["uuid"],
-            }
-        ),
-        content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}",
-    )
-
-    return entity
-
-
-async def process_projects_json(
-    json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import project data from Claude.ai projects.json format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading project data...", total=None)
-
-        # Read project data
-        data = json.loads(json_path.read_text(encoding="utf-8"))
-        progress.update(read_task, total=len(data))
-
-        # Track import counts
-        docs_imported = 0
-        prompts_imported = 0
-
-        # Process each project
-        for project in data:
-            project_dir = clean_filename(project["name"])
-
-            # Create project directories
-            docs_dir = base_path / project_dir / "docs"
-            docs_dir.mkdir(parents=True, exist_ok=True)
-
-            # Import prompt template if it exists
-            if prompt_entity := format_prompt_markdown(project):
-                file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
-                await markdown_processor.write_file(file_path, prompt_entity)
-                prompts_imported += 1
-
-            # Import project documents
-            for doc in project.get("docs", []):
-                entity = format_project_markdown(project, doc)
-                file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
-                await markdown_processor.write_file(file_path, entity)
-                docs_imported += 1
-
-            progress.update(read_task, advance=1)
-
-    return {"documents": docs_imported, "prompts": prompts_imported}
-
-
 async def get_markdown_processor() -> MarkdownProcessor:
     """Get MarkdownProcessor instance."""
     entity_parser = EntityParser(config.home)
@@ -160,30 +42,38 @@ def import_projects(
     After importing, run 'basic-memory sync' to index the new files.
     """
     try:
-        if projects_json:
-            if not projects_json.exists():
-                typer.echo(f"Error: File not found: {projects_json}", err=True)
-                raise typer.Exit(1)
-
-            # Get markdown processor
-            markdown_processor = asyncio.run(get_markdown_processor())
-
-            # Process the file
-            base_path = config.home / base_folder if base_folder else config.home
-            console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
-            results = asyncio.run(
-                process_projects_json(projects_json, base_path, markdown_processor)
-            )
-
-            # Show results
-            console.print(
-                Panel(
-                    f"[green]Import complete![/green]\n\n"
-                    f"Imported {results['documents']} project documents\n"
-                    f"Imported {results['prompts']} prompt templates",
-                    expand=False,
-                )
+        if not projects_json.exists():
+            typer.echo(f"Error: File not found: {projects_json}", err=True)
+            raise typer.Exit(1)
+
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
+
+        # Create the importer
+        importer = ClaudeProjectsImporter(config.home, markdown_processor)
+
+        # Process the file
+        base_path = config.home / base_folder if base_folder else config.home
+        console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
+
+        # Run the import
+        with projects_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, base_folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
+
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.documents} project documents\n"
+                f"Imported {result.prompts} prompt templates",
+                expand=False,
             )
+        )
 
         console.print("\nRun 'basic-memory sync' to index the new files.")