basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/cli/commands/import_chatgpt.py

@@ -2,209 +2,30 @@
 
 import asyncio
 import json
-from datetime import datetime
 from pathlib import Path
-from typing import Dict, Any, List, Annotated, Set, Optional
+from typing import Annotated, Tuple
 
-import logfire
 import typer
+from basic_memory.cli.app import import_app
+from basic_memory.config import ConfigManager, get_project_config
+from basic_memory.importers import ChatGPTImporter
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from basic_memory.services.file_service import FileService
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
-from basic_memory.cli.app import import_app
-from basic_memory.config import config
-from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_timestamp(ts: float) -> str:
-    """Format Unix timestamp for display."""
-    dt = datetime.fromtimestamp(ts)
-    return dt.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def get_message_content(message: Dict[str, Any]) -> str:
-    """Extract clean message content."""
-    if not message or "content" not in message:
-        return ""  # pragma: no cover
-
-    content = message["content"]
-    if content.get("content_type") == "text":
-        return "\n".join(content.get("parts", []))
-    elif content.get("content_type") == "code":
-        return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
-    return ""  # pragma: no cover
-
-
-def traverse_messages(
-    mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
-) -> List[Dict[str, Any]]:
-    """Traverse message tree and return messages in order."""
-    messages = []
-    node = mapping.get(root_id) if root_id else None
-
-    while node:
-        if node["id"] not in seen and node.get("message"):
-            seen.add(node["id"])
-            messages.append(node["message"])
-
-        # Follow children
-        children = node.get("children", [])
-        for child_id in children:
-            child_msgs = traverse_messages(mapping, child_id, seen)
-            messages.extend(child_msgs)
-
-        break  # Don't follow siblings
-
-    return messages
-
-
-def format_chat_markdown(
-    title: str,
-    mapping: Dict[str, Any],
-    root_id: Optional[str],
-    created_at: float,
-    modified_at: float,
-) -> str:
-    """Format chat as clean markdown."""
-
-    # Start with title
-    lines = [f"# {title}\n"]
-
-    # Traverse message tree
-    seen_msgs = set()
-    messages = traverse_messages(mapping, root_id, seen_msgs)
-
-    # Format each message
-    for msg in messages:
-        # Skip hidden messages
-        if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
-            continue
-
-        # Get author and timestamp
-        author = msg["author"]["role"].title()
-        ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
-
-        # Add message header
-        lines.append(f"### {author} ({ts})")
-
-        # Add message content
-        content = get_message_content(msg)
-        if content:
-            lines.append(content)
-
-        # Add spacing
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def format_chat_content(folder: str, conversation: Dict[str, Any]) -> EntityMarkdown:
-    """Convert chat conversation to Basic Memory entity."""
-
-    # Extract timestamps
-    created_at = conversation["create_time"]
-    modified_at = conversation["update_time"]
-
-    root_id = None
-    # Find root message
-    for node_id, node in conversation["mapping"].items():
-        if node.get("parent") is None:
-            root_id = node_id
-            break
-
-    # Generate permalink
-    date_prefix = datetime.fromtimestamp(created_at).strftime("%Y%m%d")
-    clean_title = clean_filename(conversation["title"])
-
-    # Format content
-    content = format_chat_markdown(
-        title=conversation["title"],
-        mapping=conversation["mapping"],
-        root_id=root_id,
-        created_at=created_at,
-        modified_at=modified_at,
-    )
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "conversation",
-                "title": conversation["title"],
-                "created": format_timestamp(created_at),
-                "modified": format_timestamp(modified_at),
-                "permalink": f"{folder}/{date_prefix}-{clean_title}",
-            }
-        ),
-        content=content,
-    )
-
-    return entity
-
-
-async def process_chatgpt_json(
-    json_path: Path, folder: str, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import conversations from ChatGPT JSON format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading chat data...", total=None)
-
-        # Read conversations
-        conversations = json.loads(json_path.read_text())
-        progress.update(read_task, total=len(conversations))
-
-        # Process each conversation
-        messages_imported = 0
-        chats_imported = 0
-
-        for chat in conversations:
-            # Convert to entity
-            entity = format_chat_content(folder, chat)
-
-            # Write file
-            file_path = config.home / f"{entity.frontmatter.metadata['permalink']}.md"
-            # logger.info(f"Writing file: {file_path.absolute()}")
-            await markdown_processor.write_file(file_path, entity)
-
-            # Count messages
-            msg_count = sum(
-                1
-                for node in chat["mapping"].values()
-                if node.get("message")
-                and not node.get("message", {})
-                .get("metadata", {})
-                .get("is_visually_hidden_from_conversation")
-            )
-
-            chats_imported += 1
-            messages_imported += msg_count
-            progress.update(read_task, advance=1)
-
-    return {"conversations": chats_imported, "messages": messages_imported}
-
-
-async def get_markdown_processor() -> MarkdownProcessor:
-    """Get MarkdownProcessor instance."""
+async def get_importer_dependencies() -> Tuple[MarkdownProcessor, FileService]:
+    """Get MarkdownProcessor and FileService instances for importers."""
+    config = get_project_config()
+    app_config = ConfigManager().config
     entity_parser = EntityParser(config.home)
-    return MarkdownProcessor(entity_parser)
+    markdown_processor = MarkdownProcessor(entity_parser, app_config=app_config)
+    file_service = FileService(config.home, markdown_processor, app_config=app_config)
+    return markdown_processor, file_service
 
 
 @import_app.command(name="chatgpt", help="Import conversations from ChatGPT JSON export.")
@@ -226,38 +47,41 @@ def import_chatgpt(
     After importing, run 'basic-memory sync' to index the new files.
     """
 
-    with logfire.span("import chatgpt"):  # pyright: ignore [reportGeneralTypeIssues]
-        try:
-            if conversations_json:
-                if not conversations_json.exists():
-                    typer.echo(f"Error: File not found: {conversations_json}", err=True)
-                    raise typer.Exit(1)
-
-                # Get markdown processor
-                markdown_processor = asyncio.run(get_markdown_processor())
+    try:
+        if not conversations_json.exists():  # pragma: no cover
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
 
-                # Process the file
-                base_path = config.home / folder
-                console.print(
-                    f"\nImporting chats from {conversations_json}...writing to {base_path}"
-                )
-                results = asyncio.run(
-                    process_chatgpt_json(conversations_json, folder, markdown_processor)
-                )
+        # Get importer dependencies
+        markdown_processor, file_service = asyncio.run(get_importer_dependencies())
+        config = get_project_config()
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
+
+        # Create importer and run import
+        importer = ChatGPTImporter(config.home, markdown_processor, file_service)
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
 
-                # Show results
-                console.print(
-                    Panel(
-                        f"[green]Import complete![/green]\n\n"
-                        f"Imported {results['conversations']} conversations\n"
-                        f"Containing {results['messages']} messages",
-                        expand=False,
-                    )
-                )
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
+            )
+        )
 
-            console.print("\nRun 'basic-memory sync' to index the new files.")
+        console.print("\nRun 'basic-memory sync' to index the new files.")
 
-        except Exception as e:
-            logger.error("Import failed")
-            typer.echo(f"Error during import: {e}", err=True)
-            raise typer.Exit(1)
+    except Exception as e:
+        logger.error("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)

basic_memory/cli/commands/import_claude_conversations.py

@@ -2,162 +2,30 @@
 
 import asyncio
 import json
-from datetime import datetime
 from pathlib import Path
-from typing import Dict, Any, List, Annotated
+from typing import Annotated, Tuple
 
-import logfire
 import typer
+from basic_memory.cli.app import claude_app
+from basic_memory.config import ConfigManager, get_project_config
+from basic_memory.importers.claude_conversations_importer import ClaudeConversationsImporter
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from basic_memory.services.file_service import FileService
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
-from basic_memory.cli.app import claude_app
-from basic_memory.config import config
-from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    # Remove invalid characters and convert spaces
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_timestamp(ts: str) -> str:
-    """Format ISO timestamp for display."""
-    dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
-    return dt.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def format_chat_markdown(
-    name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str
-) -> str:
-    """Format chat as clean markdown."""
-
-    # Start with frontmatter and title
-    lines = [
-        f"# {name}\n",
-    ]
-
-    # Add messages
-    for msg in messages:
-        # Format timestamp
-        ts = format_timestamp(msg["created_at"])
-
-        # Add message header
-        lines.append(f"### {msg['sender'].title()} ({ts})")
-
-        # Handle message content
-        content = msg.get("text", "")
-        if msg.get("content"):
-            content = " ".join(c.get("text", "") for c in msg["content"])
-        lines.append(content)
-
-        # Handle attachments
-        attachments = msg.get("attachments", [])
-        for attachment in attachments:
-            if "file_name" in attachment:
-                lines.append(f"\n**Attachment: {attachment['file_name']}**")
-                if "extracted_content" in attachment:
-                    lines.append("```")
-                    lines.append(attachment["extracted_content"])
-                    lines.append("```")
-
-        # Add spacing between messages
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def format_chat_content(
-    base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str
-) -> EntityMarkdown:
-    """Convert chat messages to Basic Memory entity format."""
-
-    # Generate permalink
-    date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
-    clean_title = clean_filename(name)
-    permalink = f"{base_path}/{date_prefix}-{clean_title}"
-
-    # Format content
-    content = format_chat_markdown(
-        name=name,
-        messages=messages,
-        created_at=created_at,
-        modified_at=modified_at,
-        permalink=permalink,
-    )
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "conversation",
-                "title": name,
-                "created": created_at,
-                "modified": modified_at,
-                "permalink": permalink,
-            }
-        ),
-        content=content,
-    )
-
-    return entity
-
-
-async def process_conversations_json(
-    json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import chat data from conversations2.json format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading chat data...", total=None)
-
-        # Read chat data - handle array of arrays format
-        data = json.loads(json_path.read_text())
-        conversations = [chat for chat in data]
-        progress.update(read_task, total=len(conversations))
-
-        # Process each conversation
-        messages_imported = 0
-        chats_imported = 0
-
-        for chat in conversations:
-            # Convert to entity
-            entity = format_chat_content(
-                base_path=base_path,
-                name=chat["name"],
-                messages=chat["chat_messages"],
-                created_at=chat["created_at"],
-                modified_at=chat["updated_at"],
-            )
-
-            # Write file
-            file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md")
-            await markdown_processor.write_file(file_path, entity)
-
-            chats_imported += 1
-            messages_imported += len(chat["chat_messages"])
-            progress.update(read_task, advance=1)
-
-    return {"conversations": chats_imported, "messages": messages_imported}
-
-
-async def get_markdown_processor() -> MarkdownProcessor:
-    """Get MarkdownProcessor instance."""
+async def get_importer_dependencies() -> Tuple[MarkdownProcessor, FileService]:
+    """Get MarkdownProcessor and FileService instances for importers."""
+    config = get_project_config()
+    app_config = ConfigManager().config
     entity_parser = EntityParser(config.home)
-    return MarkdownProcessor(entity_parser)
+    markdown_processor = MarkdownProcessor(entity_parser, app_config=app_config)
+    file_service = FileService(config.home, markdown_processor, app_config=app_config)
+    return markdown_processor, file_service
 
 
 @claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.")
@@ -179,35 +47,44 @@ def import_claude(
     After importing, run 'basic-memory sync' to index the new files.
     """
 
-    with logfire.span("import claude conversations"):  # pyright: ignore [reportGeneralTypeIssues]
-        try:
-            if not conversations_json.exists():
-                typer.echo(f"Error: File not found: {conversations_json}", err=True)
-                raise typer.Exit(1)
+    config = get_project_config()
+    try:
+        if not conversations_json.exists():
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
 
-            # Get markdown processor
-            markdown_processor = asyncio.run(get_markdown_processor())
+        # Get importer dependencies
+        markdown_processor, file_service = asyncio.run(get_importer_dependencies())
 
-            # Process the file
-            base_path = config.home / folder
-            console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
-            results = asyncio.run(
-                process_conversations_json(conversations_json, base_path, markdown_processor)
-            )
+        # Create the importer
+        importer = ClaudeConversationsImporter(config.home, markdown_processor, file_service)
 
-            # Show results
-            console.print(
-                Panel(
-                    f"[green]Import complete![/green]\n\n"
-                    f"Imported {results['conversations']} conversations\n"
-                    f"Containing {results['messages']} messages",
-                    expand=False,
-                )
-            )
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
 
-            console.print("\nRun 'basic-memory sync' to index the new files.")
+        # Run the import
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
 
-        except Exception as e:
-            logger.error("Import failed")
-            typer.echo(f"Error during import: {e}", err=True)
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
             raise typer.Exit(1)
+
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
+            )
+        )
+
+        console.print("\nRun 'basic-memory sync' to index the new files.")
+
+    except Exception as e:
+        logger.error("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)