basic-memory 0.0.0__py3-none-any.whl → 0.1.0__py3-none-any.whl

basic_memory/api/app.py

@@ -7,18 +7,40 @@ from fastapi.exception_handlers import http_exception_handler
 from loguru import logger
 
 from basic_memory import db
-from .routers import knowledge, search, memory, resource
+from basic_memory.api.routers import knowledge, search, memory, resource
+from basic_memory.config import config
+from basic_memory.services import DatabaseService
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Lifecycle manager for the FastAPI app."""
     logger.info("Starting Basic Memory API")
+    
+    # check the db state
+    await check_db(app)
     yield
     logger.info("Shutting down Basic Memory API")
     await db.shutdown_db()
 
 
+async def check_db(app: FastAPI):
+    logger.info("Checking database state")
+
+    # Initialize DB management service
+    db_service = DatabaseService(
+        config=config,
+    )
+
+    # Check and initialize DB if needed
+    if not await db_service.check_db():
+        raise RuntimeError("Database initialization failed")
+
+    # Clean up old backups on shutdown
+    await db_service.cleanup_backups()
+
+
+
 # Initialize FastAPI app
 app = FastAPI(
     title="Basic Memory API",

basic_memory/api/routers/memory_router.py

@@ -47,7 +47,7 @@ async def to_graph_context(context, entity_repository: EntityRepository):
 
                 return RelationSummary(
                     permalink=item.permalink,
-                    type=item.type,
+                    relation_type=item.type,
                     from_id=from_entity.permalink,
                     to_id=to_entity.permalink if to_entity else None,
                 )

basic_memory/cli/commands/__init__.py

@@ -1,9 +1,5 @@
-"""CLI commands package."""
+"""Command module exports."""
 
-from . import init, status
+from . import init, status, sync, import_memory_json
 
-
-__all__ = [
-    "init",
-    "status",
-]
+__all__ = ["init", "status", "sync", "import_memory_json.py"]

basic_memory/cli/commands/import_memory_json.py

@@ -0,0 +1,139 @@
+"""Import command for basic-memory CLI to import from JSON memory format."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Dict, Any, List
+
+import typer
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
+
+from basic_memory.cli.app import app
+from basic_memory.config import config
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation, Relation
+
+
+console = Console()
+
+async def process_memory_json(json_path: Path, base_path: Path,markdown_processor: MarkdownProcessor):
+    """Import entities from memory.json using markdown processor."""
+    
+    # First pass - collect all relations by source entity
+    entity_relations: Dict[str, List[Relation]] = {}
+    entities: Dict[str, Dict[str, Any]] = {}
+    
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        console=console,
+    ) as progress:
+        read_task = progress.add_task("Reading memory.json...", total=None)
+        
+        # First pass - collect entities and relations
+        with open(json_path) as f:
+            lines = f.readlines()
+            progress.update(read_task, total=len(lines))
+            
+            for line in lines:
+                data = json.loads(line)
+                if data["type"] == "entity":
+                    entities[data["name"]] = data
+                elif data["type"] == "relation":
+                    # Store relation with its source entity
+                    source = data.get("from") or data.get("from_id")
+                    if source not in entity_relations:
+                        entity_relations[source] = []
+                    entity_relations[source].append(
+                        Relation(
+                            type=data.get("relationType") or data.get("relation_type"),
+                            target=data.get("to") or data.get("to_id")
+                        )
+                    )
+                progress.update(read_task, advance=1)
+
+        # Second pass - create and write entities
+        write_task = progress.add_task("Creating entities...", total=len(entities))
+        
+        entities_created = 0
+        for name, entity_data in entities.items():
+            entity = EntityMarkdown(
+                frontmatter=EntityFrontmatter(
+                    metadata={
+                        "type": entity_data["entityType"],
+                        "title": name,
+                        "permalink": f"{entity_data['entityType']}/{name}"
+                    }
+                ),
+                content=f"# {name}\n",
+                observations=[
+                    Observation(content=obs) 
+                    for obs in entity_data["observations"]
+                ],
+                relations=entity_relations.get(name, [])  # Add any relations where this entity is the source
+            )
+            
+            # Let markdown processor handle writing
+            file_path = base_path / f"{entity_data['entityType']}/{name}.md"
+            await markdown_processor.write_file(file_path, entity)
+            entities_created += 1
+            progress.update(write_task, advance=1)
+    
+    return {
+        "entities": entities_created,
+        "relations": sum(len(rels) for rels in entity_relations.values())
+    }
+
+
+async def get_markdown_processor() -> MarkdownProcessor:
+    """Get MarkdownProcessor instance."""
+    entity_parser = EntityParser(config.home)
+    return MarkdownProcessor(entity_parser)
+
+
+@app.command()
+def import_json(
+    json_path: Path = typer.Argument(..., help="Path to memory.json file to import"),
+):
+    """Import entities and relations from a memory.json file.
+    
+    This command will:
+    1. Read entities and relations from the JSON file
+    2. Create markdown files for each entity
+    3. Include outgoing relations in each entity's markdown
+    
+    After importing, run 'basic-memory sync' to index the new files.
+    """
+    
+    if not json_path.exists():
+        typer.echo(f"Error: File not found: {json_path}", err=True)
+        raise typer.Exit(1)
+        
+    try:
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
+        
+        # Process the file
+        base_path = config.home
+        console.print(f"\nImporting from {json_path}...writing to {base_path}")
+        results = asyncio.run(process_memory_json(json_path, base_path, markdown_processor))
+        
+        # Show results
+        console.print(Panel(
+            f"[green]Import complete![/green]\n\n"
+            f"Created {results['entities']} entities\n"
+            f"Added {results['relations']} relations",
+            expand=False
+        ))
+        
+        console.print("\nRun 'basic-memory sync' to index the new files.")
+        
+    except Exception as e:
+        logger.exception("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)

basic_memory/cli/main.py

@@ -14,7 +14,7 @@ __all__ = ["init", "status", "sync"]
 from basic_memory.config import config
 
 
-def setup_logging(home_dir: str = config.home, log_file: str = "./basic-memory/basic-memory-tools.log"):
+def setup_logging(home_dir: str = config.home, log_file: str = ".basic-memory/basic-memory-tools.log"):
     """Configure logging for the application."""
 
     # Remove default handler and any existing handlers

basic_memory/config.py

@@ -37,7 +37,11 @@ class ProjectConfig(BaseSettings):
     @property
     def database_path(self) -> Path:
         """Get SQLite database path."""
-        return self.home / DATA_DIR_NAME / DATABASE_NAME
+        database_path = self.home / DATA_DIR_NAME / DATABASE_NAME
+        if not database_path.exists():
+            database_path.parent.mkdir(parents=True, exist_ok=True)
+            database_path.touch()
+        return database_path
 
     @field_validator("home")
     @classmethod

basic_memory/db.py

@@ -14,8 +14,7 @@ from sqlalchemy.ext.asyncio import (
     async_scoped_session,
 )
 
-from basic_memory.models import Base
-
+from basic_memory.models import Base, SCHEMA_VERSION
 
 # Module level state
 _engine: Optional[AsyncEngine] = None
@@ -69,12 +68,31 @@ async def scoped_session(
         await factory.remove()
 
 
-async def init_db(session: AsyncSession):
+async def init_db() -> None:
     """Initialize database with required tables."""
-    await session.execute(text("PRAGMA foreign_keys=ON"))
-    conn = await session.connection()
-    await conn.run_sync(Base.metadata.create_all)
-    await session.commit()
+
+    logger.info("Initializing database...")
+
+    async with scoped_session(_session_maker) as session:
+        await session.execute(text("PRAGMA foreign_keys=ON"))
+        conn = await session.connection()
+        await conn.run_sync(Base.metadata.create_all)
+        
+        await session.commit()
+
+async def drop_db():
+    """Drop all database tables."""
+    global _engine, _session_maker
+    
+    logger.info("Dropping tables...")
+    async with scoped_session(_session_maker) as session:
+        conn = await session.connection()
+        await conn.run_sync(Base.metadata.drop_all)
+        await session.commit()
+        
+    # reset global engine and session_maker
+    _engine = None
+    _session_maker = None
 
 
 async def get_or_create_db(
@@ -83,7 +101,7 @@ async def get_or_create_db(
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
     """Get or create database engine and session maker."""
     global _engine, _session_maker
-    
+
     if _engine is None:
         db_url = DatabaseType.get_db_url(db_path, db_type)
         logger.debug(f"Creating engine for db_url: {db_url}")
@@ -91,9 +109,7 @@ async def get_or_create_db(
         _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 
         # Initialize database
-        logger.debug("Initializing database...")
-        async with scoped_session(_session_maker) as db_session:
-            await init_db(db_session)
+        await init_db()
 
     return _engine, _session_maker
 
@@ -101,35 +117,38 @@ async def get_or_create_db(
 async def shutdown_db():
     """Clean up database connections."""
     global _engine, _session_maker
-    
+
     if _engine:
         await _engine.dispose()
         _engine = None
         _session_maker = None
 
 
+
 @asynccontextmanager
 async def engine_session_factory(
     db_path: Path,
-    db_type: DatabaseType = DatabaseType.FILESYSTEM,
+    db_type: DatabaseType = DatabaseType.MEMORY,
     init: bool = True,
 ) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
     """Create engine and session factory.
-    
+
     Note: This is primarily used for testing where we want a fresh database
     for each test. For production use, use get_or_create_db() instead.
     """
+
+    global _engine, _session_maker
+    
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
-    engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+    
+    _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
     try:
-        factory = async_sessionmaker(engine, expire_on_commit=False)
+        _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 
         if init:
-            logger.debug("Initializing database...")
-            async with scoped_session(factory) as db_session:
-                await init_db(db_session)
+            await init_db()
 
-        yield engine, factory
+        yield _engine, _session_maker
     finally:
-        await engine.dispose()
+        await _engine.dispose()

basic_memory/file_utils.py

@@ -170,79 +170,44 @@ def remove_frontmatter(content: str) -> str:
         raise
 
 
-def remove_frontmatter_lenient(content: str) -> str:
-    """
-    Remove frontmatter markers and anything between them without validation.
-    
-    This is a more permissive version of remove_frontmatter that doesn't
-    try to validate the YAML content. It simply removes everything between
-    the first two '---' markers if they exist.
+async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
+    """Update frontmatter fields in a file while preserving all content.
+
+    Only modifies the frontmatter section, leaving all content untouched.
+    Creates frontmatter section if none exists.
+    Returns checksum of updated file.
 
     Args:
-        content: Content that may contain frontmatter
+        path: Path to markdown file
+        updates: Dict of frontmatter fields to update
 
     Returns:
-        Content with any frontmatter markers and content removed
-    """
-    content = content.strip()
-    if not content.startswith("---"):
-        return content
-
-    # Find the second marker
-    rest = content[3:].strip()
-    if "---" not in rest:
-        return content
+        Checksum of updated file
 
-    # Split on the second marker and take everything after
-    parts = rest.split("---", 1)
-    return parts[1].strip()
-
-
-async def add_frontmatter(content: str, frontmatter: Dict[str, Any]) -> str:
-    """
-    Add YAML frontmatter to content.
-    
-    Args:
-        content: Main content text
-        frontmatter: Key-value pairs for frontmatter
-        
-    Returns:
-        Content with YAML frontmatter prepended
-        
     Raises:
-        ParseError: If YAML serialization fails
+        FileError: If file operations fail
+        ParseError: If frontmatter parsing fails
     """
     try:
-        yaml_fm = yaml.dump(frontmatter, sort_keys=False)
-        return f"---\n{yaml_fm}---\n\n{content.strip()}"
-    except yaml.YAMLError as e:
-        logger.error(f"Failed to add frontmatter: {e}")
-        raise ParseError(f"Failed to add frontmatter: {e}")
-
+        # Read current content
+        content = path.read_text()
 
-async def parse_content_with_frontmatter(content: str) -> Tuple[Dict[str, Any], str]:
-    """
-    Parse both frontmatter and content.
-
-    Args:
-        content: Text content with optional frontmatter
+        # Parse current frontmatter
+        current_fm = {}
+        if has_frontmatter(content):
+            current_fm = parse_frontmatter(content)
+            content = remove_frontmatter(content)
 
-    Returns:
-        Tuple of (frontmatter dict, content without frontmatter)
+        # Update frontmatter
+        new_fm = {**current_fm, **updates}
 
-    Raises:
-        ParseError: If parsing fails
-    """
-    try:
-        if not has_frontmatter(content):
-            return {}, content.strip()
+        # Write new file with updated frontmatter
+        yaml_fm = yaml.dump(new_fm, sort_keys=False)
+        final_content = f"---\n{yaml_fm}---\n\n{content.strip()}"
 
-        frontmatter = parse_frontmatter(content)
-        remaining = remove_frontmatter(content)
-        return frontmatter, remaining
+        await write_file_atomic(path, final_content)
+        return await compute_checksum(final_content)
 
     except Exception as e:
-        if not isinstance(e, ParseError):
-            logger.error(f"Failed to parse content with frontmatter: {e}")
-            raise ParseError(f"Failed to parse content with frontmatter: {e}")
-        raise
+        logger.error(f"Failed to update frontmatter in {path}: {e}")
+        raise FileError(f"Failed to update frontmatter: {e}")

basic_memory/markdown/__init__.py

@@ -2,6 +2,7 @@
 
 from basic_memory.file_utils import ParseError
 from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
 from basic_memory.markdown.schemas import (
     EntityMarkdown,
     EntityFrontmatter,
@@ -13,6 +14,7 @@ __all__ = [
     "EntityMarkdown",
     "EntityFrontmatter",
     "EntityParser",
+    "MarkdownProcessor",
     "Observation",
     "Relation",
     "ParseError",

basic_memory/markdown/entity_parser.py

@@ -116,7 +116,7 @@ class EntityParser:
 
         metadata = post.metadata
         metadata["title"] = post.metadata.get("title", file_path.name)
-        metadata["type"] = metadata.get("type", "note")
+        metadata["type"] = post.metadata.get("type", "note")
         metadata["tags"] = parse_tags(post.metadata.get("tags", []))
 
         # frontmatter

basic_memory/markdown/markdown_processor.py

@@ -1,17 +1,3 @@
-"""Process markdown files with structured sections.
-
-This module follows a Read -> Modify -> Write pattern for all file operations:
-1. Read entire file and parse into EntityMarkdown schema
-2. Modify the schema (add relation, update content, etc)
-3. Write entire file atomically using temp file + swap
-
-No in-place updates are performed. Each write reconstructs the entire file from the schema.
-The file format has two distinct types of content:
-1. User content - Free form text that is preserved exactly as written
-2. Structured sections - Observations and Relations that are always formatted
-   in a standard way and can be overwritten since they're tracked in our schema
-"""
-
 from pathlib import Path
 from typing import Optional
 from collections import OrderedDict
@@ -33,6 +19,8 @@ class DirtyFileError(Exception):
 
 class MarkdownProcessor:
     """Process markdown files while preserving content and structure.
+    
+    used only for import
 
     This class handles the file I/O aspects of our markdown processing. It:
     1. Uses EntityParser for reading/parsing files into our schema

basic_memory/markdown/plugins.py

@@ -68,7 +68,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:
     
     return {
         'category': category,
-        'content': ' '.join(content_parts).strip(),
+        'content': content,
         'tags': list(tags) if tags else None,
         'context': context
     }

basic_memory/markdown/schemas.py

@@ -9,15 +9,13 @@ from pydantic import BaseModel
 class Observation(BaseModel):
     """An observation about an entity."""
 
-    category: Optional[str] = None
+    category: Optional[str] = "Note"
     content: str
     tags: Optional[List[str]] = None
     context: Optional[str] = None
     
     def __str__(self) -> str:
         obs_string = f"- [{self.category}] {self.content}"
-        if self.tags:
-            obs_string += " " + " ".join(f"#{tag}" for tag in sorted(self.tags))
         if self.context:
             obs_string += f" ({self.context})"
         return obs_string

basic_memory/mcp/tools/memory.py

@@ -1,6 +1,6 @@
 """Discussion context tools for Basic Memory MCP server."""
 
-from typing import Optional, Literal
+from typing import Optional, Literal, List
 
 from loguru import logger
 
@@ -9,6 +9,7 @@ from basic_memory.mcp.server import mcp
 from basic_memory.mcp.tools.utils import call_get
 from basic_memory.schemas.memory import GraphContext, MemoryUrl, memory_url, memory_url_path, normalize_memory_url
 from basic_memory.schemas.base import TimeFrame
+from basic_memory.schemas.search import SearchItemType
 
 
 @mcp.tool(
@@ -83,7 +84,7 @@ async def build_context(
     """,
 )
 async def recent_activity(
-    type: Literal["entity", "observation", "relation"] = None,
+    type: List[Literal["entity", "observation", "relation"]] = None,
     depth: Optional[int] = 1,
     timeframe: Optional[TimeFrame] = "7d",
     max_results: int = 10,
@@ -110,6 +111,9 @@ async def recent_activity(
             - metadata: Query details and statistics
 
     Examples:
+        # Get all entities for the last 10 days (default)
+        recent_activity()
+
         # Get all entities from yesterday
         recent_activity(type=["entity"], timeframe="yesterday")
 
@@ -131,8 +135,9 @@ async def recent_activity(
         "depth": depth,
         "timeframe": timeframe,
         "max_results": max_results,
-        "type": type if type else None,
     }
+    if type: 
+        params["type"] = type
 
     response = await call_get(
         client,

basic_memory/models/__init__.py

@@ -1,12 +1,15 @@
 """Models package for basic-memory."""
 
+import basic_memory
 from basic_memory.models.base import Base
 from basic_memory.models.knowledge import Entity, Observation, Relation, ObservationCategory
 
+SCHEMA_VERSION = basic_memory.__version__ + "-" + "003"
+
 __all__ = [
-    'Base',
-    'Entity',
-    'Observation', 
-    'ObservationCategory', 
-    'Relation'
-]
+    "Base",
+    "Entity",
+    "Observation",
+    "ObservationCategory",
+    "Relation",
+]

basic_memory/models/base.py

@@ -1,9 +1,10 @@
 """Base model class for SQLAlchemy models."""
-
+from sqlalchemy import String, Integer
 from sqlalchemy.ext.asyncio import AsyncAttrs
-from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 
 class Base(AsyncAttrs, DeclarativeBase):
     """Base class for all models"""
-    pass
+    pass
+

basic_memory/repository/search_repository.py

@@ -95,14 +95,15 @@ class SearchRepository:
         permalink_match: Optional[str] = None,
         title: Optional[str] = None,
         types: List[SearchItemType] = None,
-        after_date: datetime = None,
+        after_date: Optional[datetime] = None,
         entity_types: List[str] = None,
         limit: int = 10,
     ) -> List[SearchIndexRow]:
         """Search across all indexed content with fuzzy matching."""
         conditions = []
         params = {}
-
+        order_by_clause = ""
+        
         # Handle text search for title and content
         if search_text:
             search_text = self._quote_search_term(search_text.lower().strip())
@@ -139,6 +140,9 @@ class SearchRepository:
         if after_date:
             params["after_date"] = after_date
             conditions.append("datetime(created_at) > datetime(:after_date)")
+            
+            # order by most recent first
+            order_by_clause = ", updated_at DESC"
 
         # set limit on search query
         params["limit"] = limit
@@ -165,7 +169,7 @@ class SearchRepository:
                 bm25(search_index) as score
             FROM search_index 
             WHERE {where_clause}
-            ORDER BY score ASC
+            ORDER BY score ASC {order_by_clause}
             LIMIT :limit
         """
 
@@ -197,6 +201,7 @@ class SearchRepository:
 
         #for r in results:
         #    logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
+        
         return results
 
     async def index_item(