basic-memory 0.0.0__py3-none-any.whl

basic_memory/schemas/search.py

@@ -0,0 +1,117 @@
+"""Search schemas for Basic Memory.
+
+The search system supports three primary modes:
+1. Exact permalink lookup
+2. Pattern matching with *
+3. Full-text search across content
+"""
+
+from typing import Optional, List, Union
+from datetime import datetime
+from enum import Enum
+from pydantic import BaseModel, field_validator
+
+
+class SearchItemType(str, Enum):
+    """Types of searchable items."""
+
+    ENTITY = "entity"
+    OBSERVATION = "observation"
+    RELATION = "relation"
+
+
+class SearchQuery(BaseModel):
+    """Search query parameters.
+
+    Use ONE of these primary search modes:
+    - permalink: Exact permalink match
+    - permalink_match: Path pattern with *
+    - text: Full-text search of title/content
+
+    Optionally filter results by:
+    - types: Limit to specific item types
+    - entity_types: Limit to specific entity types
+    - after_date: Only items after date
+    """
+
+    # Primary search modes (use ONE of these)
+    permalink: Optional[str] = None  # Exact permalink match
+    permalink_match: Optional[str] = None  # Exact permalink match
+    text: Optional[str] = None  # Full-text search
+    title: Optional[str] = None  # title only search
+
+    # Optional filters
+    types: Optional[List[SearchItemType]] = None  # Filter by item type
+    entity_types: Optional[List[str]] = None  # Filter by entity type
+    after_date: Optional[Union[datetime, str]] = None  # Time-based filter
+
+    @field_validator("after_date")
+    @classmethod
+    def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]:
+        """Convert datetime to ISO format if needed."""
+        if v is None:
+            return None
+        if isinstance(v, datetime):
+            return v.isoformat()
+        return v
+
+    def no_criteria(self) -> bool:
+        return (
+            self.permalink is None
+            and self.permalink_match is None
+            and self.text is None
+            and self.after_date is None
+            and self.types is None
+            and self.entity_types is None
+        )
+
+
+class SearchResult(BaseModel):
+    """Search result with score and metadata."""
+
+    id: int
+    type: SearchItemType
+    score: Optional[float] = None
+    metadata: Optional[dict] = None
+
+    # Common fields
+    permalink: Optional[str] = None
+    file_path: Optional[str] = None
+
+    # Type-specific fields
+    entity_id: Optional[int] = None  # For observations
+    category: Optional[str] = None  # For observations
+    from_id: Optional[int] = None  # For relations
+    to_id: Optional[int] = None  # For relations
+    relation_type: Optional[str] = None  # For relations
+
+
+class RelatedResult(BaseModel):
+    type: SearchItemType
+    id: int
+    title: str
+    permalink: str
+    depth: int
+    root_id: int
+    created_at: datetime
+    from_id: Optional[int] = None
+    to_id: Optional[int] = None
+    relation_type: Optional[str] = None
+    category: Optional[str] = None
+    entity_id: Optional[int] = None
+
+
+class SearchResponse(BaseModel):
+    """Wrapper for search results."""
+
+    results: List[SearchResult]
+
+
+# Schema for future advanced search endpoint
+class AdvancedSearchQuery(BaseModel):
+    """Advanced full-text search with explicit FTS5 syntax."""
+
+    query: str  # Raw FTS5 query (e.g., "foo AND bar")
+    types: Optional[List[SearchItemType]] = None
+    entity_types: Optional[List[str]] = None
+    after_date: Optional[Union[datetime, str]] = None

basic_memory/services/__init__.py

@@ -0,0 +1,11 @@
+"""Services package."""
+
+from .service import BaseService
+from .file_service import FileService
+from .entity_service import EntityService
+
+__all__ = [
+    "BaseService",
+    "FileService",
+    "EntityService",
+]

basic_memory/services/context_service.py

@@ -0,0 +1,274 @@
+"""Service for building rich context from the knowledge graph."""
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import List, Optional, Tuple
+
+from loguru import logger
+from sqlalchemy import text
+
+from basic_memory.repository.entity_repository import EntityRepository
+from basic_memory.repository.search_repository import SearchRepository
+from basic_memory.schemas.memory import MemoryUrl, memory_url_path
+from basic_memory.schemas.search import SearchItemType
+
+
+@dataclass
+class ContextResultRow:
+    type: str
+    id: int
+    title: str
+    permalink: str
+    file_path: str
+    depth: int
+    root_id: int
+    created_at: datetime
+    from_id: Optional[int] = None
+    to_id: Optional[int] = None
+    relation_type: Optional[str] = None
+    content: Optional[str] = None
+    category: Optional[str] = None
+    entity_id: Optional[int] = None
+
+
+class ContextService:
+    """Service for building rich context from memory:// URIs.
+
+    Handles three types of context building:
+    1. Direct permalink lookup - exact match on path
+    2. Pattern matching - using * wildcards
+    3. Special modes via params (e.g., 'related')
+    """
+
+    def __init__(
+        self,
+        search_repository: SearchRepository,
+        entity_repository: EntityRepository,
+    ):
+        self.search_repository = search_repository
+        self.entity_repository = entity_repository
+
+    async def build_context(
+        self,
+        memory_url: MemoryUrl = None,
+        types: List[SearchItemType] = None,
+        depth: int = 1,
+        since: Optional[datetime] = None,
+        max_results: int = 10,
+    ):
+        """Build rich context from a memory:// URI."""
+        logger.debug(
+            f"Building context for URI: '{memory_url}' depth: '{depth}' since: '{since}' max_results: '{max_results}'"
+        )
+
+        if memory_url:
+            path = memory_url_path(memory_url)
+            # Pattern matching - use search
+            if "*" in path:
+                logger.debug(f"Pattern search for '{path}'")
+                primary = await self.search_repository.search(
+                    permalink_match=path
+                )
+
+            # Direct lookup for exact path
+            else:
+                logger.debug(f"Direct lookup for '{path}'")
+                primary = await self.search_repository.search(permalink=path)
+        else:
+            logger.debug(f"Build context for '{types}'")
+            primary = await self.search_repository.search(types=types)
+
+        # Get type_id pairs for traversal
+
+        type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
+        logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
+
+        # Find related content
+        related = await self.find_related(
+            type_id_pairs, max_depth=depth, since=since, max_results=max_results
+        )
+        logger.debug(f"Found {len(related)} related results")
+        for r in related:
+            logger.debug(f"Found related {r.type}: {r.permalink}")
+
+        # Build response
+        return {
+            "primary_results": primary,
+            "related_results": related,
+            "metadata": {
+                "uri": memory_url_path(memory_url) if memory_url else None,
+                "types": types if types else None,
+                "depth": depth,
+                "timeframe": since.isoformat() if since else None,
+                "generated_at": datetime.now(timezone.utc).isoformat(),
+                "matched_results": len(primary),
+                "total_results": len(primary) + len(related),
+                "total_relations": sum(1 for r in related if r.type == SearchItemType.RELATION),
+            },
+        }
+
+    async def find_related(
+        self,
+        type_id_pairs: List[Tuple[str, int]],
+        max_depth: int = 1,
+        since: Optional[datetime] = None,
+        max_results: int = 10,
+    ) -> List[ContextResultRow]:
+        """Find items connected through relations.
+
+        Uses recursive CTE to find:
+        - Connected entities
+        - Their observations
+        - Relations that connect them
+        """
+        if not type_id_pairs:
+            return []
+
+        logger.debug(f"Finding connected items for {len(type_id_pairs)} with depth {max_depth}")
+
+        # Build the VALUES clause directly since SQLite doesn't handle parameterized IN well
+        values = ", ".join([f"('{t}', {i})" for t, i in type_id_pairs])
+
+        # Parameters for bindings
+        params = {"max_depth": max_depth, "max_results": max_results}
+        if since:
+            params["since_date"] = since.isoformat()
+
+        # Build date filter
+        date_filter = "AND base.created_at >= :since_date" if since else ""
+        r1_date_filter = "AND r.created_at >= :since_date" if since else ""
+        related_date_filter = "AND e.created_at >= :since_date" if since else ""
+
+        query = text(f"""
+WITH RECURSIVE context_graph AS (
+    -- Base case: seed items (unchanged)
+    SELECT 
+        id,
+        type,
+        title, 
+        permalink,
+        file_path,
+        from_id,
+        to_id,
+        relation_type,
+        content,
+        category,
+        entity_id,
+        0 as depth,
+        id as root_id,
+        created_at,
+        created_at as relation_date,
+        0 as is_incoming
+    FROM search_index base
+    WHERE (base.type, base.id) IN ({values})
+    {date_filter}
+
+    UNION  -- Changed from UNION ALL
+
+    -- Get relations from current entities 
+    SELECT DISTINCT
+        r.id,
+        r.type,
+        r.title,
+        r.permalink,
+        r.file_path,
+        r.from_id,
+        r.to_id,
+        r.relation_type,
+        r.content,
+        r.category,
+        r.entity_id,
+        cg.depth + 1,
+        cg.root_id,
+        r.created_at,
+        r.created_at as relation_date,
+        CASE WHEN r.from_id = cg.id THEN 0 ELSE 1 END as is_incoming
+    FROM context_graph cg
+    JOIN search_index r ON (
+        cg.type = 'entity' AND
+        r.type = 'relation' AND 
+        (r.from_id = cg.id OR r.to_id = cg.id)
+        {r1_date_filter}
+    )
+    WHERE cg.depth < :max_depth
+
+    UNION  -- Changed from UNION ALL
+
+    -- Get entities connected by relations
+    SELECT DISTINCT
+        e.id,
+        e.type,
+        e.title,
+        e.permalink,
+        e.file_path,
+        e.from_id,
+        e.to_id,
+        e.relation_type,
+        e.content,
+        e.category,
+        e.entity_id,
+        cg.depth,
+        cg.root_id,
+        e.created_at,
+        cg.relation_date,
+        cg.is_incoming
+    FROM context_graph cg
+    JOIN search_index e ON (
+        cg.type = 'relation' AND 
+        e.type = 'entity' AND
+        e.id = CASE 
+            WHEN cg.from_id = cg.id THEN cg.to_id  
+            ELSE cg.from_id                             
+        END
+        {related_date_filter}
+    )
+    WHERE cg.depth < :max_depth
+)
+SELECT DISTINCT 
+    type,
+    id,
+    title,
+    permalink,
+    file_path,
+    from_id,
+    to_id,
+    relation_type,
+    content,
+    category,
+    entity_id,
+    MIN(depth) as depth,
+    root_id,
+    created_at
+FROM context_graph
+WHERE (type, id) NOT IN ({values})
+GROUP BY
+    type, id, title, permalink, from_id, to_id,
+    relation_type, category, entity_id,
+    root_id, created_at
+ORDER BY depth, type, id
+LIMIT :max_results
+       """)
+
+        result = await self.search_repository.execute_query(query, params=params)
+        rows = result.all()
+
+        context_rows = [
+            ContextResultRow(
+                type=row.type,
+                id=row.id,
+                title=row.title,
+                permalink=row.permalink,
+                file_path=row.file_path,
+                from_id=row.from_id,
+                to_id=row.to_id,
+                relation_type=row.relation_type,
+                content=row.content,
+                category=row.category,
+                entity_id=row.entity_id,
+                depth=row.depth,
+                root_id=row.root_id,
+                created_at=row.created_at,
+            )
+            for row in rows
+        ]
+        return context_rows

basic_memory/services/entity_service.py

@@ -0,0 +1,281 @@
+"""Service for managing entities in the database."""
+
+from pathlib import Path
+from typing import Sequence, List, Optional
+
+import frontmatter
+from frontmatter import Post
+from loguru import logger
+from sqlalchemy.exc import IntegrityError
+
+from basic_memory.markdown import EntityMarkdown
+from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown
+from basic_memory.models import Entity as EntityModel, Observation, Relation
+from basic_memory.repository import ObservationRepository, RelationRepository
+from basic_memory.repository.entity_repository import EntityRepository
+from basic_memory.schemas import Entity as EntitySchema
+from basic_memory.services.exceptions import EntityNotFoundError, EntityCreationError
+from basic_memory.services import FileService
+from basic_memory.services import BaseService
+from basic_memory.services.link_resolver import LinkResolver
+from basic_memory.markdown.entity_parser import EntityParser
+
+
+class EntityService(BaseService[EntityModel]):
+    """Service for managing entities in the database."""
+
+    def __init__(
+        self,
+        entity_parser: EntityParser,
+        entity_repository: EntityRepository,
+        observation_repository: ObservationRepository,
+        relation_repository: RelationRepository,
+        file_service: FileService,
+        link_resolver: LinkResolver,
+    ):
+        super().__init__(entity_repository)
+        self.observation_repository = observation_repository
+        self.relation_repository = relation_repository
+        self.entity_parser = entity_parser
+        self.file_service = file_service
+        self.link_resolver = link_resolver
+
+    async def create_or_update_entity(self, schema: EntitySchema) -> (EntityModel, bool):
+        """Create new entity or update existing one.
+        if a new entity is created, the return value is (entity, True)
+        """
+
+        logger.debug(f"Creating or updating entity: {schema}")
+
+        # Try to find existing entity using smart resolution
+        existing = await self.link_resolver.resolve_link(schema.permalink)
+
+        if existing:
+            logger.debug(f"Found existing entity: {existing.permalink}")
+            return await self.update_entity(existing, schema), False
+        else:
+            # Create new entity
+            return await self.create_entity(schema), True
+
+    async def create_entity(self, schema: EntitySchema) -> EntityModel:
+        """Create a new entity and write to filesystem."""
+        logger.debug(f"Creating entity: {schema.permalink}")
+
+        # get file path
+        file_path = Path(schema.file_path)
+
+        if await self.file_service.exists(file_path):
+            raise EntityCreationError(
+                f"file_path {file_path} for entity {schema.permalink} already exists: {file_path}"
+            )
+
+        post = await schema_to_markdown(schema)
+
+        # write file
+        final_content = frontmatter.dumps(post, sort_keys=False)
+        checksum = await self.file_service.write_file(file_path, final_content)
+
+        # parse entity from file
+        entity_markdown = await self.entity_parser.parse_file(file_path)
+        
+        # create entity
+        created_entity = await self.create_entity_from_markdown(
+            file_path, entity_markdown
+        )
+
+        # add relations
+        entity = await self.update_entity_relations(file_path, entity_markdown)
+
+        # Set final checksum to mark complete
+        return await self.repository.update(entity.id, {"checksum": checksum})
+
+
+    async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
+        """Update an entity's content and metadata."""
+        logger.debug(f"Updating entity with permalink: {entity.permalink}")
+
+        # get file path
+        file_path = Path(entity.file_path)
+
+        post = await schema_to_markdown(schema)
+
+        # write file
+        final_content = frontmatter.dumps(post)
+        checksum = await self.file_service.write_file(file_path, final_content)
+
+        # parse entity from file
+        entity_markdown = await self.entity_parser.parse_file(file_path)
+
+        # update entity in db
+        entity = await self.update_entity_and_observations(
+            file_path, entity_markdown
+        )
+
+        # add relations
+        await self.update_entity_relations(file_path, entity_markdown)
+
+        # Set final checksum to match file
+        entity = await self.repository.update(entity.id, {"checksum": checksum})
+
+        return entity
+
+    async def delete_entity(self, permalink: str) -> bool:
+        """Delete entity and its file."""
+        logger.debug(f"Deleting entity: {permalink}")
+
+        try:
+            # Get entity first for file deletion
+            entity = await self.get_by_permalink(permalink)
+
+            # Delete file first
+            await self.file_service.delete_entity_file(entity)
+
+            # Delete from DB (this will cascade to observations/relations)
+            return await self.repository.delete(entity.id)
+
+        except EntityNotFoundError:
+            logger.info(f"Entity not found: {permalink}")
+            return True  # Already deleted
+
+        except Exception as e:
+            logger.error(f"Failed to delete entity: {e}")
+            raise
+
+    async def get_by_permalink(self, permalink: str) -> EntityModel:
+        """Get entity by type and name combination."""
+        logger.debug(f"Getting entity by permalink: {permalink}")
+        db_entity = await self.repository.get_by_permalink(permalink)
+        if not db_entity:
+            raise EntityNotFoundError(f"Entity not found: {permalink}")
+        return db_entity
+
+    async def get_all(self) -> Sequence[EntityModel]:
+        """Get all entities."""
+        return await self.repository.find_all()
+
+    async def get_entity_types(self) -> List[str]:
+        """Get list of all distinct entity types in the system."""
+        logger.debug("Getting all distinct entity types")
+        return await self.repository.get_entity_types()
+
+    async def list_entities(
+        self,
+        entity_type: Optional[str] = None,
+        sort_by: Optional[str] = "updated_at",
+        include_related: bool = False,
+    ) -> Sequence[EntityModel]:
+        """List entities with optional filtering and sorting."""
+        logger.debug(f"Listing entities: type={entity_type} sort={sort_by}")
+        return await self.repository.list_entities(entity_type=entity_type, sort_by=sort_by)
+
+    async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
+        """Get specific nodes and their relationships."""
+        logger.debug(f"Getting entities permalinks: {permalinks}")
+        return await self.repository.find_by_permalinks(permalinks)
+
+    async def delete_entity_by_file_path(self, file_path):
+        await self.repository.delete_by_file_path(file_path)
+
+    async def create_entity_from_markdown(
+        self, file_path: Path, markdown: EntityMarkdown
+    ) -> EntityModel:
+        """Create entity and observations only.
+
+        Creates the entity with null checksum to indicate sync not complete.
+        Relations will be added in second pass.
+        """
+        logger.debug(f"Creating entity: {markdown.frontmatter.title}")
+        model = entity_model_from_markdown(file_path, markdown)
+
+        # Mark as incomplete sync
+        model.checksum = None
+        return await self.add(model)
+
+    async def update_entity_and_observations(
+        self, file_path: Path | str, markdown: EntityMarkdown
+    ) -> EntityModel:
+        """Update entity fields and observations.
+
+        Updates everything except relations and sets null checksum
+        to indicate sync not complete.
+        """
+        logger.debug(f"Updating entity and observations: {file_path}")
+        file_path = str(file_path)
+
+        db_entity = await self.repository.get_by_file_path(file_path)
+        if not db_entity:
+            raise EntityNotFoundError(f"Entity not found: {file_path}")
+
+        # Clear observations for entity
+        await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
+
+        # add new observations
+        observations = [
+            Observation(
+                entity_id=db_entity.id,
+                content=obs.content,
+                category=obs.category,
+                context=obs.context,
+                tags=obs.tags,
+            )
+            for obs in markdown.observations
+        ]
+        await self.observation_repository.add_all(observations)
+
+        # update values from markdown
+        db_entity = entity_model_from_markdown(file_path, markdown, db_entity)
+
+        # checksum value is None == not finished with sync
+        db_entity.checksum = None
+        
+        # update entity
+        # checksum value is None == not finished with sync
+        return await self.repository.update(
+            db_entity.id,
+            db_entity,
+        )
+
+    async def update_entity_relations(
+        self,
+        file_path: Path | str,
+        markdown: EntityMarkdown,
+    ) -> EntityModel:
+        """Update relations for entity"""
+        logger.debug(f"Updating relations for entity: {file_path}")
+
+        file_path = str(file_path)
+        db_entity = await self.repository.get_by_file_path(file_path)
+
+        # Clear existing relations first
+        await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
+
+        # Process each relation
+        for rel in markdown.relations:
+            # Resolve the target permalink
+            target_entity = await self.link_resolver.resolve_link(
+                rel.target,
+            )
+
+            # if the target is found, store the id
+            target_id = target_entity.id if target_entity else None
+            # if the target is found, store the title, otherwise add the target for a "forward link"
+            target_name = target_entity.title if target_entity else rel.target
+
+            # Create the relation
+            relation = Relation(
+                from_id=db_entity.id,
+                to_id=target_id,
+                to_name=target_name,
+                relation_type=rel.type,
+                context=rel.context,
+            )
+            try:
+                await self.relation_repository.add(relation)
+            except IntegrityError:
+                # Unique constraint violation - relation already exists
+                logger.debug(
+                    f"Skipping duplicate relation {rel.type} from {db_entity.permalink} target: {rel.target}, type: {rel.type}"
+                )
+                continue
+
+        return await self.repository.get_by_file_path(file_path)

basic_memory/services/exceptions.py

@@ -0,0 +1,15 @@
+class FileOperationError(Exception):
+    """Raised when file operations fail"""
+
+    pass
+
+
+class EntityNotFoundError(Exception):
+    """Raised when an entity cannot be found"""
+
+    pass
+
+class EntityCreationError(Exception):
+    """Raised when an entity cannot be created"""
+
+    pass