basic-memory 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

basic_memory/repository/search_repository.py

@@ -21,31 +21,38 @@ class SearchIndexRow:
 
     id: int
     type: str
-    permalink: str
     file_path: str
-    metadata: Optional[dict] = None
 
     # date values
-    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime] = None
+    created_at: datetime
+    updated_at: datetime
+
+    permalink: Optional[str] = None
+    metadata: Optional[dict] = None
 
     # assigned in result
     score: Optional[float] = None
 
     # Type-specific fields
     title: Optional[str] = None  # entity
-    content: Optional[str] = None  # entity, observation
+    content_stems: Optional[str] = None  # entity, observation
+    content_snippet: Optional[str] = None  # entity, observation
     entity_id: Optional[int] = None  # observations
     category: Optional[str] = None  # observations
     from_id: Optional[int] = None  # relations
     to_id: Optional[int] = None  # relations
     relation_type: Optional[str] = None  # relations
 
+    @property
+    def content(self):
+        return self.content_snippet
+
     def to_insert(self):
         return {
             "id": self.id,
             "title": self.title,
-            "content": self.content,
+            "content_stems": self.content_stems,
+            "content_snippet": self.content_snippet,
             "permalink": self.permalink,
             "file_path": self.file_path,
             "type": self.type,
@@ -87,10 +94,16 @@ class SearchRepository:
         For FTS5:
         - Special characters and phrases need to be quoted
         - Terms with spaces or special chars need quotes
+        - Boolean operators (AND, OR, NOT) and parentheses are preserved
         """
         if "*" in term:
             return term
 
+        # Check for boolean operators - if present, return the term as is
+        boolean_operators = [" AND ", " OR ", " NOT ", "(", ")"]
+        if any(op in f" {term} " for op in boolean_operators):
+            return term
+
         # List of special characters that need quoting (excluding *)
         special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
 
@@ -123,9 +136,20 @@ class SearchRepository:
 
         # Handle text search for title and content
         if search_text:
-            search_text = self._prepare_search_term(search_text.strip())
-            params["text"] = search_text
-            conditions.append("(title MATCH :text OR content MATCH :text)")
+            has_boolean = any(
+                op in f" {search_text} " for op in [" AND ", " OR ", " NOT ", "(", ")"]
+            )
+
+            if has_boolean:
+                # If boolean operators are present, use the raw query
+                # No need to prepare it, FTS5 will understand the operators
+                params["text"] = search_text
+                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+            else:
+                # Standard search with term preparation
+                processed_text = self._prepare_search_term(search_text.strip())
+                params["text"] = processed_text
+                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
 
         # Handle title match search
         if title:
@@ -187,7 +211,7 @@ class SearchRepository:
                 to_id,
                 relation_type,
                 entity_id,
-                content,
+                content_snippet,
                 category,
                 created_at,
                 updated_at,
@@ -199,7 +223,7 @@ class SearchRepository:
             OFFSET :offset
         """
 
-        logger.debug(f"Search {sql} params: {params}")
+        logger.trace(f"Search {sql} params: {params}")
         async with db.scoped_session(self.session_maker) as session:
             result = await session.execute(text(sql), params)
             rows = result.fetchall()
@@ -217,7 +241,7 @@ class SearchRepository:
                 to_id=row.to_id,
                 relation_type=row.relation_type,
                 entity_id=row.entity_id,
-                content=row.content,
+                content_snippet=row.content_snippet,
                 category=row.category,
                 created_at=row.created_at,
                 updated_at=row.updated_at,
@@ -249,12 +273,12 @@ class SearchRepository:
             await session.execute(
                 text("""
                     INSERT INTO search_index (
-                        id, title, content, permalink, file_path, type, metadata,
+                        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
                         from_id, to_id, relation_type,
                         entity_id, category,
                         created_at, updated_at
                     ) VALUES (
-                        :id, :title, :content, :permalink, :file_path, :type, :metadata,
+                        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
                         :from_id, :to_id, :relation_type,
                         :entity_id, :category,
                         :created_at, :updated_at
@@ -265,6 +289,15 @@ class SearchRepository:
             logger.debug(f"indexed row {search_index_row}")
             await session.commit()
 
+    async def delete_by_entity_id(self, entity_id: int):
+        """Delete an item from the search index by entity_id."""
+        async with db.scoped_session(self.session_maker) as session:
+            await session.execute(
+                text("DELETE FROM search_index WHERE entity_id = :entity_id"),
+                {"entity_id": entity_id},
+            )
+            await session.commit()
+
     async def delete_by_permalink(self, permalink: str):
         """Delete an item from the search index."""
         async with db.scoped_session(self.session_maker) as session:

basic_memory/schemas/__init__.py

@@ -37,11 +37,11 @@ from basic_memory.schemas.response import (
     DeleteEntitiesResponse,
 )
 
-# Discovery and analytics models
-from basic_memory.schemas.discovery import (
-    EntityTypeList,
-    ObservationCategoryList,
-    TypedEntityList,
+from basic_memory.schemas.project_info import (
+    ProjectStatistics,
+    ActivityMetrics,
+    SystemStatus,
+    ProjectInfoResponse,
 )
 
 # For convenient imports, export all models
@@ -66,8 +66,9 @@ __all__ = [
     "DeleteEntitiesResponse",
     # Delete Operations
     "DeleteEntitiesRequest",
-    # Discovery and Analytics
-    "EntityTypeList",
-    "ObservationCategoryList",
-    "TypedEntityList",
+    # Project Info
+    "ProjectStatistics",
+    "ActivityMetrics",
+    "SystemStatus",
+    "ProjectInfoResponse",
 ]

basic_memory/schemas/base.py

@@ -159,7 +159,10 @@ class Entity(BaseModel):
     @property
     def file_path(self):
         """Get the file path for this entity based on its permalink."""
-        return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
+        if self.content_type == "text/markdown":
+            return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
+        else:
+            return f"{self.folder}/{self.title}" if self.folder else self.title
 
     @property
     def permalink(self) -> Permalink:

basic_memory/schemas/memory.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
 from basic_memory.schemas.search import SearchItemType
 
 
-def normalize_memory_url(url: str) -> str:
+def normalize_memory_url(url: str | None) -> str:
     """Normalize a MemoryUrl string.
 
     Args:
@@ -24,6 +24,9 @@ def normalize_memory_url(url: str) -> str:
         >>> normalize_memory_url("memory://specs/search")
         'memory://specs/search'
     """
+    if not url:
+        return ""
+
     clean_path = url.removeprefix("memory://")
     return f"memory://{clean_path}"
 
@@ -59,8 +62,9 @@ class EntitySummary(BaseModel):
     """Simplified entity representation."""
 
     type: str = "entity"
-    permalink: str
+    permalink: Optional[str]
     title: str
+    content: Optional[str] = None
     file_path: str
     created_at: datetime
 
@@ -69,19 +73,25 @@ class RelationSummary(BaseModel):
     """Simplified relation representation."""
 
     type: str = "relation"
+    title: str
+    file_path: str
     permalink: str
     relation_type: str
-    from_id: str
-    to_id: Optional[str] = None
+    from_entity: str
+    to_entity: Optional[str] = None
+    created_at: datetime
 
 
 class ObservationSummary(BaseModel):
     """Simplified observation representation."""
 
     type: str = "observation"
+    title: str
+    file_path: str
     permalink: str
     category: str
     content: str
+    created_at: datetime
 
 
 class MemoryMetadata(BaseModel):

basic_memory/schemas/project_info.py

@@ -0,0 +1,96 @@
+"""Schema for project info response."""
+
+from datetime import datetime
+from typing import Dict, List, Optional, Any
+
+from pydantic import Field, BaseModel
+
+
+class ProjectStatistics(BaseModel):
+    """Statistics about the current project."""
+
+    # Basic counts
+    total_entities: int = Field(description="Total number of entities in the knowledge base")
+    total_observations: int = Field(description="Total number of observations across all entities")
+    total_relations: int = Field(description="Total number of relations between entities")
+    total_unresolved_relations: int = Field(
+        description="Number of relations with unresolved targets"
+    )
+
+    # Entity counts by type
+    entity_types: Dict[str, int] = Field(
+        description="Count of entities by type (e.g., note, conversation)"
+    )
+
+    # Observation counts by category
+    observation_categories: Dict[str, int] = Field(
+        description="Count of observations by category (e.g., tech, decision)"
+    )
+
+    # Relation counts by type
+    relation_types: Dict[str, int] = Field(
+        description="Count of relations by type (e.g., implements, relates_to)"
+    )
+
+    # Graph metrics
+    most_connected_entities: List[Dict[str, Any]] = Field(
+        description="Entities with the most relations, including their titles and permalinks"
+    )
+    isolated_entities: int = Field(description="Number of entities with no relations")
+
+
+class ActivityMetrics(BaseModel):
+    """Activity metrics for the current project."""
+
+    # Recent activity
+    recently_created: List[Dict[str, Any]] = Field(
+        description="Recently created entities with timestamps"
+    )
+    recently_updated: List[Dict[str, Any]] = Field(
+        description="Recently updated entities with timestamps"
+    )
+
+    # Growth over time (last 6 months)
+    monthly_growth: Dict[str, Dict[str, int]] = Field(
+        description="Monthly growth statistics for entities, observations, and relations"
+    )
+
+
+class SystemStatus(BaseModel):
+    """System status information."""
+
+    # Version information
+    version: str = Field(description="Basic Memory version")
+
+    # Database status
+    database_path: str = Field(description="Path to the SQLite database")
+    database_size: str = Field(description="Size of the database in human-readable format")
+
+    # Watch service status
+    watch_status: Optional[Dict[str, Any]] = Field(
+        default=None, description="Watch service status information (if running)"
+    )
+
+    # System information
+    timestamp: datetime = Field(description="Timestamp when the information was collected")
+
+
+class ProjectInfoResponse(BaseModel):
+    """Response for the project_info tool."""
+
+    # Project configuration
+    project_name: str = Field(description="Name of the current project")
+    project_path: str = Field(description="Path to the current project files")
+    available_projects: Dict[str, str] = Field(
+        description="Map of configured project names to paths"
+    )
+    default_project: str = Field(description="Name of the default project")
+
+    # Statistics
+    statistics: ProjectStatistics = Field(description="Statistics about the knowledge base")
+
+    # Activity metrics
+    activity: ActivityMetrics = Field(description="Activity and growth metrics")
+
+    # System status
+    system: SystemStatus = Field(description="System and service status information")

basic_memory/schemas/search.py

@@ -11,6 +11,8 @@ from datetime import datetime
 from enum import Enum
 from pydantic import BaseModel, field_validator
 
+from basic_memory.schemas.base import Permalink
+
 
 class SearchItemType(str, Enum):
     """Types of searchable items."""
@@ -26,18 +28,24 @@ class SearchQuery(BaseModel):
     Use ONE of these primary search modes:
     - permalink: Exact permalink match
     - permalink_match: Path pattern with *
-    - text: Full-text search of title/content
+    - text: Full-text search of title/content (supports boolean operators: AND, OR, NOT)
 
     Optionally filter results by:
     - types: Limit to specific item types
     - entity_types: Limit to specific entity types
     - after_date: Only items after date
+
+    Boolean search examples:
+    - "python AND flask" - Find items with both terms
+    - "python OR django" - Find items with either term
+    - "python NOT django" - Find items with python but not django
+    - "(python OR flask) AND web" - Use parentheses for grouping
     """
 
     # Primary search modes (use ONE of these)
     permalink: Optional[str] = None  # Exact permalink match
-    permalink_match: Optional[str] = None  # Exact permalink match
-    text: Optional[str] = None  # Full-text search
+    permalink_match: Optional[str] = None  # Glob permalink match
+    text: Optional[str] = None  # Full-text search (now supports boolean operators)
     title: Optional[str] = None  # title only search
 
     # Optional filters
@@ -57,60 +65,48 @@ class SearchQuery(BaseModel):
         return (
             self.permalink is None
             and self.permalink_match is None
+            and self.title is None
             and self.text is None
             and self.after_date is None
             and self.types is None
             and self.entity_types is None
         )
 
+    def has_boolean_operators(self) -> bool:
+        """Check if the text query contains boolean operators (AND, OR, NOT)."""
+        if not self.text:  # pragma: no cover
+            return False
+
+        # Check for common boolean operators with correct word boundaries
+        # to avoid matching substrings like "GRAND" containing "AND"
+        boolean_patterns = [" AND ", " OR ", " NOT ", "(", ")"]
+        text = f" {self.text} "  # Add spaces to ensure we match word boundaries
+        return any(pattern in text for pattern in boolean_patterns)
+
 
 class SearchResult(BaseModel):
     """Search result with score and metadata."""
 
-    id: int
+    title: str
     type: SearchItemType
     score: float
-    permalink: str
+    entity: Optional[Permalink]
+    permalink: Optional[str]
+    content: Optional[str] = None
     file_path: str
 
     metadata: Optional[dict] = None
 
     # Type-specific fields
-    entity_id: Optional[int] = None  # For observations
     category: Optional[str] = None  # For observations
-    from_id: Optional[int] = None  # For relations
-    to_id: Optional[int] = None  # For relations
+    from_entity: Optional[Permalink] = None  # For relations
+    to_entity: Optional[Permalink] = None  # For relations
     relation_type: Optional[str] = None  # For relations
 
 
-class RelatedResult(BaseModel):
-    type: SearchItemType
-    id: int
-    title: str
-    permalink: str
-    depth: int
-    root_id: int
-    created_at: datetime
-    from_id: Optional[int] = None
-    to_id: Optional[int] = None
-    relation_type: Optional[str] = None
-    category: Optional[str] = None
-    entity_id: Optional[int] = None
-
-
 class SearchResponse(BaseModel):
     """Wrapper for search results."""
 
     results: List[SearchResult]
     current_page: int
     page_size: int
-
-
-# Schema for future advanced search endpoint
-class AdvancedSearchQuery(BaseModel):
-    """Advanced full-text search with explicit FTS5 syntax."""
-
-    query: str  # Raw FTS5 query (e.g., "foo AND bar")
-    types: Optional[List[SearchItemType]] = None
-    entity_types: Optional[List[str]] = None
-    after_date: Optional[Union[datetime, str]] = None

basic_memory/services/context_service.py

@@ -165,7 +165,7 @@ class ContextService:
                 from_id,
                 to_id,
                 relation_type,
-                content,
+                content_snippet as content,
                 category,
                 entity_id,
                 0 as depth,
@@ -189,7 +189,7 @@ class ContextService:
                 r.from_id,
                 r.to_id,
                 r.relation_type,
-                r.content,
+                r.content_snippet as content,
                 r.category,
                 r.entity_id,
                 cg.depth + 1,
@@ -218,7 +218,7 @@ class ContextService:
                 e.from_id,
                 e.to_id,
                 e.relation_type,
-                e.content,
+                e.content_snippet as content,
                 e.category,
                 e.entity_id,
                 cg.depth + 1,  -- Increment depth for entities

basic_memory/services/entity_service.py

@@ -124,17 +124,19 @@ class EntityService(BaseService[EntityModel]):
         entity_markdown = await self.entity_parser.parse_file(file_path)
 
         # create entity
-        await self.create_entity_from_markdown(file_path, entity_markdown)
+        created = await self.create_entity_from_markdown(file_path, entity_markdown)
 
         # add relations
-        entity = await self.update_entity_relations(file_path, entity_markdown)
+        entity = await self.update_entity_relations(created.file_path, entity_markdown)
 
         # Set final checksum to mark complete
         return await self.repository.update(entity.id, {"checksum": checksum})
 
     async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
         """Update an entity's content and metadata."""
-        logger.debug(f"Updating entity with permalink: {entity.permalink}")
+        logger.debug(
+            f"Updating entity with permalink: {entity.permalink} content-type: {schema.content_type}"
+        )
 
         # Convert file path string to Path
         file_path = Path(entity.file_path)
@@ -142,7 +144,7 @@ class EntityService(BaseService[EntityModel]):
         post = await schema_to_markdown(schema)
 
         # write file
-        final_content = frontmatter.dumps(post)
+        final_content = frontmatter.dumps(post, sort_keys=False)
         checksum = await self.file_service.write_file(file_path, final_content)
 
         # parse entity from file
@@ -152,20 +154,31 @@ class EntityService(BaseService[EntityModel]):
         entity = await self.update_entity_and_observations(file_path, entity_markdown)
 
         # add relations
-        await self.update_entity_relations(file_path, entity_markdown)
+        await self.update_entity_relations(str(file_path), entity_markdown)
 
         # Set final checksum to match file
         entity = await self.repository.update(entity.id, {"checksum": checksum})
 
         return entity
 
-    async def delete_entity(self, permalink: str) -> bool:
+    async def delete_entity(self, permalink_or_id: str | int) -> bool:
         """Delete entity and its file."""
-        logger.debug(f"Deleting entity: {permalink}")
+        logger.debug(f"Deleting entity: {permalink_or_id}")
 
         try:
             # Get entity first for file deletion
-            entity = await self.get_by_permalink(permalink)
+            if isinstance(permalink_or_id, str):
+                entity = await self.get_by_permalink(permalink_or_id)
+            else:
+                entities = await self.get_entities_by_id([permalink_or_id])
+                if len(entities) != 1:  # pragma: no cover
+                    logger.error(
+                        "Entity lookup error", entity_id=permalink_or_id, found_count=len(entities)
+                    )
+                    raise ValueError(
+                        f"Expected 1 entity with ID {permalink_or_id}, got {len(entities)}"
+                    )
+                entity = entities[0]
 
             # Delete file first
             await self.file_service.delete_entity_file(entity)
@@ -174,7 +187,7 @@ class EntityService(BaseService[EntityModel]):
             return await self.repository.delete(entity.id)
 
         except EntityNotFoundError:
-            logger.info(f"Entity not found: {permalink}")
+            logger.info(f"Entity not found: {permalink_or_id}")
             return True  # Already deleted
 
     async def get_by_permalink(self, permalink: str) -> EntityModel:
@@ -256,13 +269,13 @@ class EntityService(BaseService[EntityModel]):
 
     async def update_entity_relations(
         self,
-        file_path: Path,
+        path: str,
         markdown: EntityMarkdown,
     ) -> EntityModel:
         """Update relations for entity"""
-        logger.debug(f"Updating relations for entity: {file_path}")
+        logger.debug(f"Updating relations for entity: {path}")
 
-        db_entity = await self.repository.get_by_file_path(str(file_path))
+        db_entity = await self.repository.get_by_file_path(path)
 
         # Clear existing relations first
         await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
@@ -296,4 +309,4 @@ class EntityService(BaseService[EntityModel]):
                 )
                 continue
 
-        return await self.repository.get_by_file_path(str(file_path))
+        return await self.repository.get_by_file_path(path)