basic-memory 0.7.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/cli/commands/db.py

@@ -2,27 +2,43 @@
 
 import asyncio
 
-import logfire
 import typer
 from loguru import logger
 
-from basic_memory.alembic import migrations
+from basic_memory import db
 from basic_memory.cli.app import app
+from basic_memory.config import ConfigManager, BasicMemoryConfig, save_basic_memory_config
 
 
 @app.command()
 def reset(
-    reindex: bool = typer.Option(False, "--reindex", help="Rebuild indices from filesystem"),
+    reindex: bool = typer.Option(False, "--reindex", help="Rebuild db index from filesystem"),
 ):  # pragma: no cover
     """Reset database (drop all tables and recreate)."""
-    with logfire.span("reset"):  # pyright: ignore [reportGeneralTypeIssues]
-        if typer.confirm("This will delete all data in your db. Are you sure?"):
-            logger.info("Resetting database...")
-            asyncio.run(migrations.reset_database())
+    if typer.confirm("This will delete all data in your db. Are you sure?"):
+        logger.info("Resetting database...")
+        config_manager = ConfigManager()
+        app_config = config_manager.config
+        # Get database path
+        db_path = app_config.app_database_path
 
-            if reindex:
-                # Import and run sync
-                from basic_memory.cli.commands.sync import sync
+        # Delete the database file if it exists
+        if db_path.exists():
+            db_path.unlink()
+            logger.info(f"Database file deleted: {db_path}")
 
-                logger.info("Rebuilding search index from filesystem...")
-                sync(watch=False)  # pyright: ignore
+        # Reset project configuration
+        config = BasicMemoryConfig()
+        save_basic_memory_config(config_manager.config_file, config)
+        logger.info("Project configuration reset to default")
+
+        # Create a new empty database
+        asyncio.run(db.run_migrations(app_config))
+        logger.info("Database reset complete")
+
+        if reindex:
+            # Run database sync directly
+            from basic_memory.cli.commands.command_utils import run_sync
+
+            logger.info("Rebuilding search index from filesystem...")
+            asyncio.run(run_sync(project=None))

basic_memory/cli/commands/import_chatgpt.py

@@ -2,207 +2,24 @@
 
 import asyncio
 import json
-from datetime import datetime
 from pathlib import Path
-from typing import Dict, Any, List, Annotated, Set, Optional
+from typing import Annotated
 
-import logfire
 import typer
+from basic_memory.cli.app import import_app
+from basic_memory.config import get_project_config
+from basic_memory.importers import ChatGPTImporter
+from basic_memory.markdown import EntityParser, MarkdownProcessor
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
-from basic_memory.cli.app import import_app
-from basic_memory.config import config
-from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_timestamp(ts: float) -> str:
-    """Format Unix timestamp for display."""
-    dt = datetime.fromtimestamp(ts)
-    return dt.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def get_message_content(message: Dict[str, Any]) -> str:
-    """Extract clean message content."""
-    if not message or "content" not in message:
-        return ""  # pragma: no cover
-
-    content = message["content"]
-    if content.get("content_type") == "text":
-        return "\n".join(content.get("parts", []))
-    elif content.get("content_type") == "code":
-        return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
-    return ""  # pragma: no cover
-
-
-def traverse_messages(
-    mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
-) -> List[Dict[str, Any]]:
-    """Traverse message tree and return messages in order."""
-    messages = []
-    node = mapping.get(root_id) if root_id else None
-
-    while node:
-        if node["id"] not in seen and node.get("message"):
-            seen.add(node["id"])
-            messages.append(node["message"])
-
-        # Follow children
-        children = node.get("children", [])
-        for child_id in children:
-            child_msgs = traverse_messages(mapping, child_id, seen)
-            messages.extend(child_msgs)
-
-        break  # Don't follow siblings
-
-    return messages
-
-
-def format_chat_markdown(
-    title: str,
-    mapping: Dict[str, Any],
-    root_id: Optional[str],
-    created_at: float,
-    modified_at: float,
-) -> str:
-    """Format chat as clean markdown."""
-
-    # Start with title
-    lines = [f"# {title}\n"]
-
-    # Traverse message tree
-    seen_msgs = set()
-    messages = traverse_messages(mapping, root_id, seen_msgs)
-
-    # Format each message
-    for msg in messages:
-        # Skip hidden messages
-        if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
-            continue
-
-        # Get author and timestamp
-        author = msg["author"]["role"].title()
-        ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
-
-        # Add message header
-        lines.append(f"### {author} ({ts})")
-
-        # Add message content
-        content = get_message_content(msg)
-        if content:
-            lines.append(content)
-
-        # Add spacing
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def format_chat_content(folder: str, conversation: Dict[str, Any]) -> EntityMarkdown:
-    """Convert chat conversation to Basic Memory entity."""
-
-    # Extract timestamps
-    created_at = conversation["create_time"]
-    modified_at = conversation["update_time"]
-
-    root_id = None
-    # Find root message
-    for node_id, node in conversation["mapping"].items():
-        if node.get("parent") is None:
-            root_id = node_id
-            break
-
-    # Generate permalink
-    date_prefix = datetime.fromtimestamp(created_at).strftime("%Y%m%d")
-    clean_title = clean_filename(conversation["title"])
-
-    # Format content
-    content = format_chat_markdown(
-        title=conversation["title"],
-        mapping=conversation["mapping"],
-        root_id=root_id,
-        created_at=created_at,
-        modified_at=modified_at,
-    )
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "conversation",
-                "title": conversation["title"],
-                "created": format_timestamp(created_at),
-                "modified": format_timestamp(modified_at),
-                "permalink": f"{folder}/{date_prefix}-{clean_title}",
-            }
-        ),
-        content=content,
-    )
-
-    return entity
-
-
-async def process_chatgpt_json(
-    json_path: Path, folder: str, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import conversations from ChatGPT JSON format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading chat data...", total=None)
-
-        # Read conversations
-        conversations = json.loads(json_path.read_text())
-        progress.update(read_task, total=len(conversations))
-
-        # Process each conversation
-        messages_imported = 0
-        chats_imported = 0
-
-        for chat in conversations:
-            # Convert to entity
-            entity = format_chat_content(folder, chat)
-
-            # Write file
-            file_path = config.home / f"{entity.frontmatter.metadata['permalink']}.md"
-            # logger.info(f"Writing file: {file_path.absolute()}")
-            await markdown_processor.write_file(file_path, entity)
-
-            # Count messages
-            msg_count = sum(
-                1
-                for node in chat["mapping"].values()
-                if node.get("message")
-                and not node.get("message", {})
-                .get("metadata", {})
-                .get("is_visually_hidden_from_conversation")
-            )
-
-            chats_imported += 1
-            messages_imported += msg_count
-            progress.update(read_task, advance=1)
-
-    return {"conversations": chats_imported, "messages": messages_imported}
-
-
 async def get_markdown_processor() -> MarkdownProcessor:
     """Get MarkdownProcessor instance."""
+    config = get_project_config()
     entity_parser = EntityParser(config.home)
     return MarkdownProcessor(entity_parser)
 
@@ -226,38 +43,41 @@ def import_chatgpt(
     After importing, run 'basic-memory sync' to index the new files.
     """
 
-    with logfire.span("import chatgpt"):  # pyright: ignore [reportGeneralTypeIssues]
-        try:
-            if conversations_json:
-                if not conversations_json.exists():
-                    typer.echo(f"Error: File not found: {conversations_json}", err=True)
-                    raise typer.Exit(1)
-
-                # Get markdown processor
-                markdown_processor = asyncio.run(get_markdown_processor())
+    try:
+        if not conversations_json.exists():  # pragma: no cover
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
 
-                # Process the file
-                base_path = config.home / folder
-                console.print(
-                    f"\nImporting chats from {conversations_json}...writing to {base_path}"
-                )
-                results = asyncio.run(
-                    process_chatgpt_json(conversations_json, folder, markdown_processor)
-                )
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
+        config = get_project_config()
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
+
+        # Create importer and run import
+        importer = ChatGPTImporter(config.home, markdown_processor)
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
 
-                # Show results
-                console.print(
-                    Panel(
-                        f"[green]Import complete![/green]\n\n"
-                        f"Imported {results['conversations']} conversations\n"
-                        f"Containing {results['messages']} messages",
-                        expand=False,
-                    )
-                )
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
+            )
+        )
 
-            console.print("\nRun 'basic-memory sync' to index the new files.")
+        console.print("\nRun 'basic-memory sync' to index the new files.")
 
-        except Exception as e:
-            logger.error("Import failed")
-            typer.echo(f"Error during import: {e}", err=True)
-            raise typer.Exit(1)
+    except Exception as e:
+        logger.error("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)

basic_memory/cli/commands/import_claude_conversations.py

@@ -2,160 +2,24 @@
 
 import asyncio
 import json
-from datetime import datetime
 from pathlib import Path
-from typing import Dict, Any, List, Annotated
+from typing import Annotated
 
-import logfire
 import typer
+from basic_memory.cli.app import claude_app
+from basic_memory.config import get_project_config
+from basic_memory.importers.claude_conversations_importer import ClaudeConversationsImporter
+from basic_memory.markdown import EntityParser, MarkdownProcessor
 from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
-
-from basic_memory.cli.app import claude_app
-from basic_memory.config import config
-from basic_memory.markdown import EntityParser, MarkdownProcessor
-from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
 
 console = Console()
 
 
-def clean_filename(text: str) -> str:
-    """Convert text to safe filename."""
-    # Remove invalid characters and convert spaces
-    clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
-    return clean
-
-
-def format_timestamp(ts: str) -> str:
-    """Format ISO timestamp for display."""
-    dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
-    return dt.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def format_chat_markdown(
-    name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str
-) -> str:
-    """Format chat as clean markdown."""
-
-    # Start with frontmatter and title
-    lines = [
-        f"# {name}\n",
-    ]
-
-    # Add messages
-    for msg in messages:
-        # Format timestamp
-        ts = format_timestamp(msg["created_at"])
-
-        # Add message header
-        lines.append(f"### {msg['sender'].title()} ({ts})")
-
-        # Handle message content
-        content = msg.get("text", "")
-        if msg.get("content"):
-            content = " ".join(c.get("text", "") for c in msg["content"])
-        lines.append(content)
-
-        # Handle attachments
-        attachments = msg.get("attachments", [])
-        for attachment in attachments:
-            if "file_name" in attachment:
-                lines.append(f"\n**Attachment: {attachment['file_name']}**")
-                if "extracted_content" in attachment:
-                    lines.append("```")
-                    lines.append(attachment["extracted_content"])
-                    lines.append("```")
-
-        # Add spacing between messages
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def format_chat_content(
-    base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str
-) -> EntityMarkdown:
-    """Convert chat messages to Basic Memory entity format."""
-
-    # Generate permalink
-    date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
-    clean_title = clean_filename(name)
-    permalink = f"{base_path}/{date_prefix}-{clean_title}"
-
-    # Format content
-    content = format_chat_markdown(
-        name=name,
-        messages=messages,
-        created_at=created_at,
-        modified_at=modified_at,
-        permalink=permalink,
-    )
-
-    # Create entity
-    entity = EntityMarkdown(
-        frontmatter=EntityFrontmatter(
-            metadata={
-                "type": "conversation",
-                "title": name,
-                "created": created_at,
-                "modified": modified_at,
-                "permalink": permalink,
-            }
-        ),
-        content=content,
-    )
-
-    return entity
-
-
-async def process_conversations_json(
-    json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
-) -> Dict[str, int]:
-    """Import chat data from conversations2.json format."""
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        console=console,
-    ) as progress:
-        read_task = progress.add_task("Reading chat data...", total=None)
-
-        # Read chat data - handle array of arrays format
-        data = json.loads(json_path.read_text())
-        conversations = [chat for chat in data]
-        progress.update(read_task, total=len(conversations))
-
-        # Process each conversation
-        messages_imported = 0
-        chats_imported = 0
-
-        for chat in conversations:
-            # Convert to entity
-            entity = format_chat_content(
-                base_path=base_path,
-                name=chat["name"],
-                messages=chat["chat_messages"],
-                created_at=chat["created_at"],
-                modified_at=chat["updated_at"],
-            )
-
-            # Write file
-            file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md")
-            await markdown_processor.write_file(file_path, entity)
-
-            chats_imported += 1
-            messages_imported += len(chat["chat_messages"])
-            progress.update(read_task, advance=1)
-
-    return {"conversations": chats_imported, "messages": messages_imported}
-
-
 async def get_markdown_processor() -> MarkdownProcessor:
     """Get MarkdownProcessor instance."""
+    config = get_project_config()
     entity_parser = EntityParser(config.home)
     return MarkdownProcessor(entity_parser)
 
@@ -179,35 +43,44 @@ def import_claude(
     After importing, run 'basic-memory sync' to index the new files.
     """
 
-    with logfire.span("import claude conversations"):  # pyright: ignore [reportGeneralTypeIssues]
-        try:
-            if not conversations_json.exists():
-                typer.echo(f"Error: File not found: {conversations_json}", err=True)
-                raise typer.Exit(1)
+    config = get_project_config()
+    try:
+        if not conversations_json.exists():
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
 
-            # Get markdown processor
-            markdown_processor = asyncio.run(get_markdown_processor())
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
 
-            # Process the file
-            base_path = config.home / folder
-            console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
-            results = asyncio.run(
-                process_conversations_json(conversations_json, base_path, markdown_processor)
-            )
+        # Create the importer
+        importer = ClaudeConversationsImporter(config.home, markdown_processor)
 
-            # Show results
-            console.print(
-                Panel(
-                    f"[green]Import complete![/green]\n\n"
-                    f"Imported {results['conversations']} conversations\n"
-                    f"Containing {results['messages']} messages",
-                    expand=False,
-                )
-            )
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
 
-            console.print("\nRun 'basic-memory sync' to index the new files.")
+        # Run the import
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
 
-        except Exception as e:
-            logger.error("Import failed")
-            typer.echo(f"Error during import: {e}", err=True)
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
             raise typer.Exit(1)
+
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
+            )
+        )
+
+        console.print("\nRun 'basic-memory sync' to index the new files.")
+
+    except Exception as e:
+        logger.error("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)