basic-memory 0.0.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

basic_memory/repository/search_repository.py

@@ -70,6 +70,8 @@ class SearchRepository:
 
     async def init_search_index(self):
         """Create or recreate the search index."""
+        
+        logger.info("Initializing search index")
         async with db.scoped_session(self.session_maker) as session:
             await session.execute(CREATE_SEARCH_INDEX)
             await session.commit()
@@ -95,14 +97,15 @@ class SearchRepository:
         permalink_match: Optional[str] = None,
         title: Optional[str] = None,
         types: List[SearchItemType] = None,
-        after_date: datetime = None,
+        after_date: Optional[datetime] = None,
         entity_types: List[str] = None,
         limit: int = 10,
     ) -> List[SearchIndexRow]:
         """Search across all indexed content with fuzzy matching."""
         conditions = []
         params = {}
-
+        order_by_clause = ""
+        
         # Handle text search for title and content
         if search_text:
             search_text = self._quote_search_term(search_text.lower().strip())
@@ -139,6 +142,9 @@ class SearchRepository:
         if after_date:
             params["after_date"] = after_date
             conditions.append("datetime(created_at) > datetime(:after_date)")
+            
+            # order by most recent first
+            order_by_clause = ", updated_at DESC"
 
         # set limit on search query
         params["limit"] = limit
@@ -165,7 +171,7 @@ class SearchRepository:
                 bm25(search_index) as score
             FROM search_index 
             WHERE {where_clause}
-            ORDER BY score ASC
+            ORDER BY score ASC {order_by_clause}
             LIMIT :limit
         """
 
@@ -197,6 +203,7 @@ class SearchRepository:
 
         #for r in results:
         #    logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
+        
         return results
 
     async def index_item(

basic_memory/schemas/base.py

@@ -181,6 +181,9 @@ class Entity(BaseModel):
     - Optional relations to other entities
     - Optional description for high-level overview
     """
+    
+    # private field to override permalink
+    _permalink: Optional[str] = None
 
     title: str
     content: Optional[str] = None
@@ -199,8 +202,8 @@ class Entity(BaseModel):
     
     @property
     def permalink(self) -> PathId:
-        """Get the path ID in format {snake_case_title}."""
-        return generate_permalink(self.file_path)
+        """Get a url friendly path}."""
+        return self._permalink or generate_permalink(self.file_path)
 
     @model_validator(mode="after")
     @classmethod

basic_memory/schemas/memory.py

@@ -59,6 +59,7 @@ def memory_url_path(url: memory_url) -> str:
 class EntitySummary(BaseModel):
     """Simplified entity representation."""
 
+    type: str = "entity"
     permalink: str
     title: str
     file_path: str
@@ -68,8 +69,9 @@ class EntitySummary(BaseModel):
 class RelationSummary(BaseModel):
     """Simplified relation representation."""
 
+    type: str = "relation"
     permalink: str
-    type: str
+    relation_type: str
     from_id: str
     to_id: Optional[str] = None
 
@@ -77,6 +79,7 @@ class RelationSummary(BaseModel):
 class ObservationSummary(BaseModel):
     """Simplified observation representation."""
 
+    type: str = "observation"
     permalink: str
     category: str
     content: str

basic_memory/services/__init__.py

@@ -1,5 +1,5 @@
 """Services package."""
-
+from .database_service import DatabaseService
 from .service import BaseService
 from .file_service import FileService
 from .entity_service import EntityService
@@ -8,4 +8,5 @@ __all__ = [
     "BaseService",
     "FileService",
     "EntityService",
+    "DatabaseService"
 ]

basic_memory/services/context_service.py

@@ -76,7 +76,7 @@ class ContextService:
                 primary = await self.search_repository.search(permalink=path)
         else:
             logger.debug(f"Build context for '{types}'")
-            primary = await self.search_repository.search(types=types)
+            primary = await self.search_repository.search(types=types, after_date=since)
 
         # Get type_id pairs for traversal
 

basic_memory/services/database_service.py

@@ -0,0 +1,159 @@
+"""Service for managing database lifecycle and schema validation."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Tuple, List
+
+from alembic.runtime.migration import MigrationContext
+from alembic.autogenerate import compare_metadata
+from loguru import logger
+from sqlalchemy import MetaData
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from basic_memory import db
+from basic_memory.config import ProjectConfig
+from basic_memory.models import Base
+from basic_memory.repository.search_repository import SearchRepository
+
+
+async def check_schema_matches_models(session: AsyncSession) -> Tuple[bool, List[str]]:
+    """Check if database schema matches SQLAlchemy models.
+    
+    Returns:
+        tuple[bool, list[str]]: (matches, list of differences)
+    """
+    # Get current DB schema via migration context
+    conn = await session.connection()
+
+    def _compare_schemas(connection):
+        context = MigrationContext.configure(connection)
+        return compare_metadata(context, Base.metadata)
+
+    # Run comparison in sync context
+    differences = await conn.run_sync(_compare_schemas)
+    
+    if not differences:
+        return True, []
+        
+    # Format differences into readable messages
+    diff_messages = []
+    for diff in differences:
+        if diff[0] == 'add_table':
+            diff_messages.append(f"Missing table: {diff[1].name}")
+        elif diff[0] == 'remove_table':
+            diff_messages.append(f"Extra table: {diff[1].name}")
+        elif diff[0] == 'add_column':
+            diff_messages.append(f"Missing column: {diff[3]} in table {diff[2]}")
+        elif diff[0] == 'remove_column':
+            diff_messages.append(f"Extra column: {diff[3]} in table {diff[2]}")
+        elif diff[0] == 'modify_type':
+            diff_messages.append(f"Column type mismatch: {diff[3]} in table {diff[2]}")
+            
+    return False, diff_messages
+
+
+class DatabaseService:
+    """Manages database lifecycle including schema validation and backups."""
+
+    def __init__(
+        self,
+        config: ProjectConfig,
+        db_type: db.DatabaseType = db.DatabaseType.FILESYSTEM,
+    ):
+        self.config = config
+        self.db_path = Path(config.database_path)
+        self.db_type = db_type
+
+    async def create_backup(self) -> Optional[Path]:
+        """Create backup of existing database file.
+
+        Returns:
+            Optional[Path]: Path to backup file if created, None if no DB exists
+        """
+        if self.db_type == db.DatabaseType.MEMORY:
+            return None  # Skip backups for in-memory DB
+
+        if not self.db_path.exists():
+            return None
+
+        # Create backup with timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        backup_path = self.db_path.with_suffix(f".{timestamp}.backup")
+
+        try:
+            self.db_path.rename(backup_path)
+            logger.info(f"Created database backup: {backup_path}")
+            
+            # make a new empty file
+            self.db_path.touch()
+            return backup_path
+        except Exception as e:
+            logger.error(f"Failed to create database backup: {e}")
+            return None
+
+    async def initialize_db(self):
+        """Initialize database with current schema."""
+        logger.info("Initializing database...")
+
+        if self.db_type == db.DatabaseType.FILESYSTEM:
+            await self.create_backup()
+
+        # Drop existing tables if any
+        await db.drop_db()
+
+        # Create tables with current schema
+        await db.get_or_create_db(
+            db_path=self.db_path,
+            db_type=self.db_type
+        )
+
+        logger.info("Database initialized with current schema")
+
+    async def check_db(self) -> bool:
+        """Check database state and rebuild if schema doesn't match models.
+        
+        Returns:
+            bool: True if DB is ready for use, False if initialization failed
+        """
+        try:
+            _, session_maker = await db.get_or_create_db(
+                db_path=self.db_path,
+                db_type=self.db_type
+            )
+            async with db.scoped_session(session_maker) as db_session:
+                # Check actual schema matches
+                matches, differences = await check_schema_matches_models(db_session)
+                if not matches:
+                    logger.warning("Database schema does not match models:")
+                    for diff in differences:
+                        logger.warning(f"  {diff}")
+                    logger.info("Rebuilding database to match current models...")
+                    await self.initialize_db()                    
+                    return True
+                    
+                logger.info("Database schema matches models")
+                return True
+
+        except Exception as e:
+            logger.error(f"Database initialization failed: {e}")
+            return False
+
+    async def cleanup_backups(self, keep_count: int = 5):
+        """Clean up old database backups, keeping the N most recent."""
+        if self.db_type == db.DatabaseType.MEMORY:
+            return  # Skip cleanup for in-memory DB
+
+        backup_pattern = "*.backup"  # Use relative pattern
+        backups = sorted(
+            self.db_path.parent.glob(backup_pattern),
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+
+        # Remove old backups
+        for backup in backups[keep_count:]:
+            try:
+                backup.unlink()
+                logger.debug(f"Removed old backup: {backup}")
+            except Exception as e:
+                logger.error(f"Failed to remove backup {backup}: {e}")

basic_memory/services/entity_service.py

@@ -19,6 +19,7 @@ from basic_memory.services import FileService
 from basic_memory.services import BaseService
 from basic_memory.services.link_resolver import LinkResolver
 from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.utils import generate_permalink
 
 
 class EntityService(BaseService[EntityModel]):
@@ -40,6 +41,51 @@ class EntityService(BaseService[EntityModel]):
         self.file_service = file_service
         self.link_resolver = link_resolver
 
+    async def resolve_permalink(
+            self,
+            file_path: Path,
+            markdown: Optional[EntityMarkdown] = None
+    ) -> str:
+        """Get or generate unique permalink for an entity.
+
+        Priority:
+        1. If markdown has permalink and it's not used by another file -> use as is
+        2. If markdown has permalink but it's used by another file -> make unique
+        3. For existing files, keep current permalink from db
+        4. Generate new unique permalink from file path
+        """
+        file_path = str(file_path)
+
+        # If markdown has explicit permalink, try to validate it
+        if markdown and markdown.frontmatter.permalink:
+            desired_permalink = markdown.frontmatter.permalink
+            existing = await self.repository.get_by_permalink(desired_permalink)
+
+            # If no conflict or it's our own file, use as is
+            if not existing or existing.file_path == file_path:
+                return desired_permalink
+
+        # For existing files, try to find current permalink
+        existing = await self.repository.get_by_file_path(file_path)
+        if existing:
+            return existing.permalink
+
+        # New file - generate permalink
+        if markdown and markdown.frontmatter.permalink:
+            desired_permalink = markdown.frontmatter.permalink
+        else:
+            desired_permalink = generate_permalink(file_path)
+
+        # Make unique if needed
+        permalink = desired_permalink
+        suffix = 1
+        while await self.repository.get_by_permalink(permalink):
+            permalink = f"{desired_permalink}-{suffix}"
+            suffix += 1
+            logger.debug(f"creating unique permalink: {permalink}")
+
+        return permalink
+    
     async def create_or_update_entity(self, schema: EntitySchema) -> (EntityModel, bool):
         """Create new entity or update existing one.
         if a new entity is created, the return value is (entity, True)
@@ -66,9 +112,13 @@ class EntityService(BaseService[EntityModel]):
 
         if await self.file_service.exists(file_path):
             raise EntityCreationError(
-                f"file_path {file_path} for entity {schema.permalink} already exists: {file_path}"
+                f"file for entity {schema.folder}/{schema.title} already exists: {file_path}"
             )
 
+        # Get unique permalink
+        permalink = await self.resolve_permalink(schema.permalink or file_path)
+        schema._permalink = permalink
+
         post = await schema_to_markdown(schema)
 
         # write file
@@ -184,7 +234,7 @@ class EntityService(BaseService[EntityModel]):
         Creates the entity with null checksum to indicate sync not complete.
         Relations will be added in second pass.
         """
-        logger.debug(f"Creating entity: {markdown.frontmatter.title}")
+        logger.debug(f"Creating entity: {markdown.frontmatter.title}")        
         model = entity_model_from_markdown(file_path, markdown)
 
         # Mark as incomplete sync

basic_memory/services/file_service.py

@@ -35,7 +35,6 @@ class FileService:
         """Generate absolute filesystem path for entity."""
         return self.base_path / f"{entity.file_path}"
 
-    # TODO move to tests
     async def write_entity_file(
         self,
         entity: EntityModel,

basic_memory/sync/sync_service.py

@@ -4,7 +4,9 @@ from pathlib import Path
 from typing import Dict
 
 from loguru import logger
+from sqlalchemy.exc import IntegrityError
 
+from basic_memory import file_utils
 from basic_memory.markdown import EntityParser, EntityMarkdown
 from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService
@@ -92,8 +94,32 @@ class SyncService:
         # First pass: Create/update entities
         # entities will have a null checksum to indicate they are not complete
         for file_path, entity_markdown in parsed_entities.items():
+
+            # Get unique permalink and update markdown if needed
+            permalink = await self.entity_service.resolve_permalink(
+                file_path,
+                markdown=entity_markdown
+            )
+
+            if permalink != entity_markdown.frontmatter.permalink:
+                # Add/update permalink in frontmatter
+                logger.info(f"Adding permalink '{permalink}' to file: {file_path}")
+
+                # update markdown
+                entity_markdown.frontmatter.metadata["permalink"] = permalink
+                
+                # update file frontmatter
+                updated_checksum = await file_utils.update_frontmatter(
+                    directory / file_path,
+                    {"permalink": permalink}
+                )
+
+                # Update checksum in changes report since file was modified
+                changes.checksums[file_path] = updated_checksum
+            
             # if the file is new, create an entity
             if file_path in changes.new:
+                # Create entity with final permalink
                 logger.debug(f"Creating new entity_markdown: {file_path}")
                 await self.entity_service.create_entity_from_markdown(
                     file_path, entity_markdown
@@ -128,10 +154,14 @@ class SyncService:
             # check we found a link that is not the source
             if target_entity and target_entity.id != relation.from_id:
                 logger.debug(f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}")
-                await self.relation_repository.update(relation.id, {
-                    "to_id": target_entity.id,
-                    "to_name": target_entity.title  # Update to actual title
-                })
+
+                try:
+                    await self.relation_repository.update(relation.id, {
+                        "to_id": target_entity.id,
+                        "to_name": target_entity.title  # Update to actual title
+                    })
+                except IntegrityError as e:
+                        logger.info(f"Ignoring duplicate relation {relation}")
 
                 # update search index
                 await self.search_service.index_entity(target_entity)