basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/repository/search_repository.py

@@ -1,108 +1,35 @@
-"""Repository for search operations."""
+"""Repository for search operations.
+
+This module provides the search repository interface.
+The actual repository implementations are backend-specific:
+- SQLiteSearchRepository: Uses FTS5 virtual tables
+- PostgresSearchRepository: Uses tsvector/tsquery with GIN indexes
+"""
 
-import json
-import time
-from dataclasses import dataclass
 from datetime import datetime
-from typing import List, Optional, Any, Dict
+from typing import List, Optional, Protocol
 
-from loguru import logger
-from sqlalchemy import text, Executable, Result
+from sqlalchemy import Result
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
-from basic_memory import db
-from basic_memory.models.search import CREATE_SEARCH_INDEX
+from basic_memory.config import ConfigManager, DatabaseBackend
+from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
+from basic_memory.repository.search_index_row import SearchIndexRow
+from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
 from basic_memory.schemas.search import SearchItemType
 
 
-@dataclass
-class SearchIndexRow:
-    """Search result with score and metadata."""
-
-    id: int
-    type: str
-    permalink: str
-    file_path: str
-    metadata: Optional[dict] = None
-
-    # date values
-    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime] = None
-
-    # assigned in result
-    score: Optional[float] = None
-
-    # Type-specific fields
-    title: Optional[str] = None  # entity
-    content: Optional[str] = None  # entity, observation
-    entity_id: Optional[int] = None  # observations
-    category: Optional[str] = None  # observations
-    from_id: Optional[int] = None  # relations
-    to_id: Optional[int] = None  # relations
-    relation_type: Optional[str] = None  # relations
-
-    def to_insert(self):
-        return {
-            "id": self.id,
-            "title": self.title,
-            "content": self.content,
-            "permalink": self.permalink,
-            "file_path": self.file_path,
-            "type": self.type,
-            "metadata": json.dumps(self.metadata),
-            "from_id": self.from_id,
-            "to_id": self.to_id,
-            "relation_type": self.relation_type,
-            "entity_id": self.entity_id,
-            "category": self.category,
-            "created_at": self.created_at if self.created_at else None,
-            "updated_at": self.updated_at if self.updated_at else None,
-        }
-
-
-class SearchRepository:
-    """Repository for search index operations."""
-
-    def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
-        self.session_maker = session_maker
+class SearchRepository(Protocol):
+    """Protocol defining the search repository interface.
 
-    async def init_search_index(self):
-        """Create or recreate the search index."""
-        logger.info("Initializing search index")
-        try:
-            async with db.scoped_session(self.session_maker) as session:
-                await session.execute(CREATE_SEARCH_INDEX)
-                await session.commit()
-        except Exception as e:  # pragma: no cover
-            logger.error(f"Error initializing search index: {e}")
-            raise e
+    Both SQLite and Postgres implementations must satisfy this protocol.
+    """
 
-    def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
-        """Prepare a search term for FTS5 query.
+    project_id: int
 
-        Args:
-            term: The search term to prepare
-            is_prefix: Whether to add prefix search capability (* suffix)
-
-        For FTS5:
-        - Special characters and phrases need to be quoted
-        - Terms with spaces or special chars need quotes
-        """
-        if "*" in term:
-            return term
-
-        # List of special characters that need quoting (excluding *)
-        special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
-
-        # Check if term contains any special characters
-        needs_quotes = any(c in term for c in special_chars)
-
-        if needs_quotes:
-            # If the term already contains quotes, escape them and add a wildcard
-            term = term.replace('"', '""')
-            term = f'"{term}"*'
-
-        return term
+    async def init_search_index(self) -> None:
+        """Initialize the search index schema."""
+        ...
 
     async def search(
         self,
@@ -110,181 +37,65 @@ class SearchRepository:
         permalink: Optional[str] = None,
         permalink_match: Optional[str] = None,
         title: Optional[str] = None,
-        types: Optional[List[SearchItemType]] = None,
+        types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
-        entity_types: Optional[List[str]] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:
-        """Search across all indexed content with fuzzy matching."""
-        conditions = []
-        params = {}
-        order_by_clause = ""
-
-        # Handle text search for title and content
-        if search_text:
-            search_text = self._prepare_search_term(search_text.strip())
-            params["text"] = search_text
-            conditions.append("(title MATCH :text OR content MATCH :text)")
-
-        # Handle title match search
-        if title:
-            title_text = self._prepare_search_term(title.strip())
-            params["text"] = title_text
-            conditions.append("title MATCH :text")
-
-        # Handle permalink exact search
-        if permalink:
-            params["permalink"] = permalink
-            conditions.append("permalink = :permalink")
-
-        # Handle permalink match search, supports *
-        if permalink_match:
-            # Clean and prepare permalink for FTS5 GLOB match
-            permalink_text = self._prepare_search_term(
-                permalink_match.lower().strip(), is_prefix=False
-            )
-            params["permalink"] = permalink_text
-            if "*" in permalink_match:
-                conditions.append("permalink GLOB :permalink")
-            else:
-                conditions.append("permalink MATCH :permalink")
-
-        # Handle type filter
-        if types:
-            type_list = ", ".join(f"'{t.value}'" for t in types)
-            conditions.append(f"type IN ({type_list})")
-
-        # Handle entity type filter
-        if entity_types:
-            entity_type_list = ", ".join(f"'{t}'" for t in entity_types)
-            conditions.append(f"json_extract(metadata, '$.entity_type') IN ({entity_type_list})")
-
-        # Handle date filter using datetime() for proper comparison
-        if after_date:
-            params["after_date"] = after_date
-            conditions.append("datetime(created_at) > datetime(:after_date)")
-
-            # order by most recent first
-            order_by_clause = ", updated_at DESC"
-
-        # set limit on search query
-        params["limit"] = limit
-        params["offset"] = offset
-
-        # Build WHERE clause
-        where_clause = " AND ".join(conditions) if conditions else "1=1"
-
-        sql = f"""
-            SELECT 
-                id, 
-                title, 
-                permalink,
-                file_path,
-                type,
-                metadata,
-                from_id,
-                to_id,
-                relation_type,
-                entity_id,
-                content,
-                category,
-                created_at,
-                updated_at,
-                bm25(search_index) as score
-            FROM search_index 
-            WHERE {where_clause}
-            ORDER BY score ASC {order_by_clause}
-            LIMIT :limit
-            OFFSET :offset
-        """
-
-        logger.debug(f"Search {sql} params: {params}")
-        async with db.scoped_session(self.session_maker) as session:
-            result = await session.execute(text(sql), params)
-            rows = result.fetchall()
-
-        results = [
-            SearchIndexRow(
-                id=row.id,
-                title=row.title,
-                permalink=row.permalink,
-                file_path=row.file_path,
-                type=row.type,
-                score=row.score,
-                metadata=json.loads(row.metadata),
-                from_id=row.from_id,
-                to_id=row.to_id,
-                relation_type=row.relation_type,
-                entity_id=row.entity_id,
-                content=row.content,
-                category=row.category,
-                created_at=row.created_at,
-                updated_at=row.updated_at,
-            )
-            for row in rows
-        ]
-
-        logger.debug(f"Found {len(results)} search results")
-        for r in results:
-            logger.debug(
-                f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}"
-            )
-
-        return results
-
-    async def index_item(
-        self,
-        search_index_row: SearchIndexRow,
-    ):
-        """Index or update a single item."""
-        async with db.scoped_session(self.session_maker) as session:
-            # Delete existing record if any
-            await session.execute(
-                text("DELETE FROM search_index WHERE permalink = :permalink"),
-                {"permalink": search_index_row.permalink},
-            )
-
-            # Insert new record
-            await session.execute(
-                text("""
-                    INSERT INTO search_index (
-                        id, title, content, permalink, file_path, type, metadata,
-                        from_id, to_id, relation_type,
-                        entity_id, category,
-                        created_at, updated_at
-                    ) VALUES (
-                        :id, :title, :content, :permalink, :file_path, :type, :metadata,
-                        :from_id, :to_id, :relation_type,
-                        :entity_id, :category,
-                        :created_at, :updated_at
-                    )
-                """),
-                search_index_row.to_insert(),
-            )
-            logger.debug(f"indexed row {search_index_row}")
-            await session.commit()
-
-    async def delete_by_permalink(self, permalink: str):
-        """Delete an item from the search index."""
-        async with db.scoped_session(self.session_maker) as session:
-            await session.execute(
-                text("DELETE FROM search_index WHERE permalink = :permalink"),
-                {"permalink": permalink},
-            )
-            await session.commit()
-
-    async def execute_query(
-        self,
-        query: Executable,
-        params: Dict[str, Any],
-    ) -> Result[Any]:
-        """Execute a query asynchronously."""
-        # logger.debug(f"Executing query: {query}, params: {params}")
-        async with db.scoped_session(self.session_maker) as session:
-            start_time = time.perf_counter()
-            result = await session.execute(query, params)
-            end_time = time.perf_counter()
-            elapsed_time = end_time - start_time
-            logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
-            return result
+        """Search across indexed content."""
+        ...
+
+    async def index_item(self, search_index_row: SearchIndexRow) -> None:
+        """Index a single item."""
+        ...
+
+    async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
+        """Index multiple items in a batch."""
+        ...
+
+    async def delete_by_permalink(self, permalink: str) -> None:
+        """Delete item by permalink."""
+        ...
+
+    async def delete_by_entity_id(self, entity_id: int) -> None:
+        """Delete items by entity ID."""
+        ...
+
+    async def execute_query(self, query, params: dict) -> Result:
+        """Execute a raw SQL query."""
+        ...
+
+
+def create_search_repository(
+    session_maker: async_sessionmaker[AsyncSession],
+    project_id: int,
+    database_backend: Optional[DatabaseBackend] = None,
+) -> SearchRepository:
+    """Factory function to create the appropriate search repository based on database backend.
+
+    Args:
+        session_maker: SQLAlchemy async session maker
+        project_id: Project ID for the repository
+        database_backend: Optional explicit backend. If not provided, reads from ConfigManager.
+            Prefer passing explicitly from composition roots.
+
+    Returns:
+        SearchRepository: Backend-appropriate search repository instance
+    """
+    # Prefer explicit parameter; fall back to ConfigManager for backwards compatibility
+    if database_backend is None:
+        config = ConfigManager().config
+        database_backend = config.database_backend
+
+    if database_backend == DatabaseBackend.POSTGRES:  # pragma: no cover
+        return PostgresSearchRepository(session_maker, project_id=project_id)  # pragma: no cover
+    else:
+        return SQLiteSearchRepository(session_maker, project_id=project_id)
+
+
+__all__ = [
+    "SearchRepository",
+    "SearchIndexRow",
+    "create_search_repository",
+]

basic_memory/repository/search_repository_base.py

@@ -0,0 +1,241 @@
+"""Abstract base class for search repository implementations."""
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+from loguru import logger
+from sqlalchemy import Executable, Result, text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from basic_memory import db
+from basic_memory.schemas.search import SearchItemType
+from basic_memory.repository.search_index_row import SearchIndexRow
+
+
+class SearchRepositoryBase(ABC):
+    """Abstract base class for backend-specific search repository implementations.
+
+    This class defines the common interface that all search repositories must implement,
+    regardless of whether they use SQLite FTS5 or Postgres tsvector for full-text search.
+
+    Concrete implementations:
+    - SQLiteSearchRepository: Uses FTS5 virtual tables with MATCH queries
+    - PostgresSearchRepository: Uses tsvector/tsquery with GIN indexes
+    """
+
+    def __init__(self, session_maker: async_sessionmaker[AsyncSession], project_id: int):
+        """Initialize with session maker and project_id filter.
+
+        Args:
+            session_maker: SQLAlchemy session maker
+            project_id: Project ID to filter all operations by
+
+        Raises:
+            ValueError: If project_id is None or invalid
+        """
+        if project_id is None or project_id <= 0:  # pragma: no cover
+            raise ValueError("A valid project_id is required for SearchRepository")
+
+        self.session_maker = session_maker
+        self.project_id = project_id
+
+    @abstractmethod
+    async def init_search_index(self) -> None:
+        """Create or recreate the search index.
+
+        Backend-specific implementations:
+        - SQLite: CREATE VIRTUAL TABLE using FTS5
+        - Postgres: CREATE TABLE with tsvector column and GIN indexes
+        """
+        pass
+
+    @abstractmethod
+    def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
+        """Prepare a search term for backend-specific query syntax.
+
+        Args:
+            term: The search term to prepare
+            is_prefix: Whether to add prefix search capability
+
+        Returns:
+            Formatted search term for the backend
+
+        Backend-specific implementations:
+        - SQLite: Quotes FTS5 special characters, adds * wildcards
+        - Postgres: Converts to tsquery syntax with :* prefix operator
+        """
+        pass
+
+    @abstractmethod
+    async def search(
+        self,
+        search_text: Optional[str] = None,
+        permalink: Optional[str] = None,
+        permalink_match: Optional[str] = None,
+        title: Optional[str] = None,
+        types: Optional[List[str]] = None,
+        after_date: Optional[datetime] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
+        limit: int = 10,
+        offset: int = 0,
+    ) -> List[SearchIndexRow]:
+        """Search across all indexed content.
+
+        Args:
+            search_text: Full-text search across title and content
+            permalink: Exact permalink match
+            permalink_match: Permalink pattern match (supports *)
+            title: Title search
+            types: Filter by entity types (from metadata.entity_type)
+            after_date: Filter by created_at > after_date
+            search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
+            limit: Maximum results to return
+            offset: Number of results to skip
+
+        Returns:
+            List of SearchIndexRow results with relevance scores
+
+        Backend-specific implementations:
+        - SQLite: Uses MATCH operator and bm25() for scoring
+        - Postgres: Uses @@ operator and ts_rank() for scoring
+        """
+        pass
+
+    async def index_item(self, search_index_row: SearchIndexRow) -> None:
+        """Index or update a single item.
+
+        This implementation is shared across backends as it uses standard SQL INSERT.
+        """
+
+        async with db.scoped_session(self.session_maker) as session:
+            # Delete existing record if any
+            await session.execute(
+                text(
+                    "DELETE FROM search_index WHERE permalink = :permalink AND project_id = :project_id"
+                ),
+                {"permalink": search_index_row.permalink, "project_id": self.project_id},
+            )
+
+            # When using text() raw SQL, always serialize JSON to string
+            # Both SQLite (TEXT) and Postgres (JSONB) accept JSON strings in raw SQL
+            # The database driver/column type will handle conversion
+            insert_data = search_index_row.to_insert(serialize_json=True)
+            insert_data["project_id"] = self.project_id
+
+            # Insert new record
+            await session.execute(
+                text("""
+                    INSERT INTO search_index (
+                        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+                        from_id, to_id, relation_type,
+                        entity_id, category,
+                        created_at, updated_at,
+                        project_id
+                    ) VALUES (
+                        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
+                        :from_id, :to_id, :relation_type,
+                        :entity_id, :category,
+                        :created_at, :updated_at,
+                        :project_id
+                    )
+                """),
+                insert_data,
+            )
+            logger.debug(f"indexed row {search_index_row}")
+            await session.commit()
+
+    async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
+        """Index multiple items in a single batch operation.
+
+        This implementation is shared across backends as it uses standard SQL INSERT.
+
+        Note: This method assumes that any existing records for the entity_id
+        have already been deleted (typically via delete_by_entity_id).
+
+        Args:
+            search_index_rows: List of SearchIndexRow objects to index
+        """
+
+        if not search_index_rows:  # pragma: no cover
+            return  # pragma: no cover
+
+        async with db.scoped_session(self.session_maker) as session:
+            # When using text() raw SQL, always serialize JSON to string
+            # Both SQLite (TEXT) and Postgres (JSONB) accept JSON strings in raw SQL
+            # The database driver/column type will handle conversion
+            insert_data_list = []
+            for row in search_index_rows:
+                insert_data = row.to_insert(serialize_json=True)
+                insert_data["project_id"] = self.project_id
+                insert_data_list.append(insert_data)
+
+            # Batch insert all records using executemany
+            await session.execute(
+                text("""
+                    INSERT INTO search_index (
+                        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+                        from_id, to_id, relation_type,
+                        entity_id, category,
+                        created_at, updated_at,
+                        project_id
+                    ) VALUES (
+                        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
+                        :from_id, :to_id, :relation_type,
+                        :entity_id, :category,
+                        :created_at, :updated_at,
+                        :project_id
+                    )
+                """),
+                insert_data_list,
+            )
+            logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
+            await session.commit()
+
+    async def delete_by_entity_id(self, entity_id: int) -> None:
+        """Delete all search index entries for an entity.
+
+        This implementation is shared across backends as it uses standard SQL DELETE.
+        """
+        async with db.scoped_session(self.session_maker) as session:
+            await session.execute(
+                text(
+                    "DELETE FROM search_index WHERE entity_id = :entity_id AND project_id = :project_id"
+                ),
+                {"entity_id": entity_id, "project_id": self.project_id},
+            )
+            await session.commit()
+
+    async def delete_by_permalink(self, permalink: str) -> None:
+        """Delete a search index entry by permalink.
+
+        This implementation is shared across backends as it uses standard SQL DELETE.
+        """
+        async with db.scoped_session(self.session_maker) as session:
+            await session.execute(
+                text(
+                    "DELETE FROM search_index WHERE permalink = :permalink AND project_id = :project_id"
+                ),
+                {"permalink": permalink, "project_id": self.project_id},
+            )
+            await session.commit()
+
+    async def execute_query(
+        self,
+        query: Executable,
+        params: Dict[str, Any],
+    ) -> Result[Any]:
+        """Execute a query asynchronously.
+
+        This implementation is shared across backends for utility query execution.
+        """
+        import time
+
+        async with db.scoped_session(self.session_maker) as session:
+            start_time = time.perf_counter()
+            result = await session.execute(query, params)
+            end_time = time.perf_counter()
+            elapsed_time = end_time - start_time
+            logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
+            return result