basic-memory 0.12.2__py3-none-any.whl → 0.13.0__py3-none-any.whl

basic_memory/repository/repository.py

@@ -1,6 +1,6 @@
 """Base repository implementation."""
 
-from typing import Type, Optional, Any, Sequence, TypeVar, List
+from typing import Type, Optional, Any, Sequence, TypeVar, List, Dict
 
 from loguru import logger
 from sqlalchemy import (
@@ -27,13 +27,30 @@ T = TypeVar("T", bound=Base)
 class Repository[T: Base]:
     """Base repository implementation with generic CRUD operations."""
 
-    def __init__(self, session_maker: async_sessionmaker[AsyncSession], Model: Type[T]):
+    def __init__(
+        self,
+        session_maker: async_sessionmaker[AsyncSession],
+        Model: Type[T],
+        project_id: Optional[int] = None,
+    ):
         self.session_maker = session_maker
+        self.project_id = project_id
         if Model:
             self.Model = Model
             self.mapper = inspect(self.Model).mapper
             self.primary_key: Column[Any] = self.mapper.primary_key[0]
             self.valid_columns = [column.key for column in self.mapper.columns]
+            # Check if this model has a project_id column
+            self.has_project_id = "project_id" in self.valid_columns
+
+    def _set_project_id_if_needed(self, model: T) -> None:
+        """Set project_id on model if needed and available."""
+        if (
+            self.has_project_id
+            and self.project_id is not None
+            and getattr(model, "project_id", None) is None
+        ):
+            setattr(model, "project_id", self.project_id)
 
     def get_model_data(self, entity_data):
         model_data = {
@@ -41,6 +58,19 @@ class Repository[T: Base]:
         }
         return model_data
 
+    def _add_project_filter(self, query: Select) -> Select:
+        """Add project_id filter to query if applicable.
+
+        Args:
+            query: The SQLAlchemy query to modify
+
+        Returns:
+            Updated query with project filter if applicable
+        """
+        if self.has_project_id and self.project_id is not None:
+            query = query.filter(getattr(self.Model, "project_id") == self.project_id)
+        return query
+
     async def select_by_id(self, session: AsyncSession, entity_id: int) -> Optional[T]:
         """Select an entity by ID using an existing session."""
         query = (
@@ -48,6 +78,9 @@ class Repository[T: Base]:
             .filter(self.primary_key == entity_id)
             .options(*self.get_load_options())
         )
+        # Add project filter if applicable
+        query = self._add_project_filter(query)
+
         result = await session.execute(query)
         return result.scalars().one_or_none()
 
@@ -56,6 +89,9 @@ class Repository[T: Base]:
         query = (
             select(self.Model).where(self.primary_key.in_(ids)).options(*self.get_load_options())
         )
+        # Add project filter if applicable
+        query = self._add_project_filter(query)
+
         result = await session.execute(query)
         return result.scalars().all()
 
@@ -66,6 +102,9 @@ class Repository[T: Base]:
         :return: the added model instance
         """
         async with db.scoped_session(self.session_maker) as session:
+            # Set project_id if applicable and not already set
+            self._set_project_id_if_needed(model)
+
             session.add(model)
             await session.flush()
 
@@ -89,6 +128,10 @@ class Repository[T: Base]:
         :return: the added models instances
         """
         async with db.scoped_session(self.session_maker) as session:
+            # set the project id if not present in models
+            for model in models:
+                self._set_project_id_if_needed(model)
+
             session.add_all(models)
             await session.flush()
 
@@ -104,7 +147,10 @@ class Repository[T: Base]:
         """
         if not entities:
             entities = (self.Model,)
-        return select(*entities)
+        query = select(*entities)
+
+        # Add project filter if applicable
+        return self._add_project_filter(query)
 
     async def find_all(self, skip: int = 0, limit: Optional[int] = None) -> Sequence[T]:
         """Fetch records from the database with pagination."""
@@ -112,6 +158,9 @@ class Repository[T: Base]:
 
         async with db.scoped_session(self.session_maker) as session:
             query = select(self.Model).offset(skip).options(*self.get_load_options())
+            # Add project filter if applicable
+            query = self._add_project_filter(query)
+
             if limit:
                 query = query.limit(limit)
 
@@ -143,9 +192,9 @@ class Repository[T: Base]:
         entity = result.scalars().one_or_none()
 
         if entity:
-            logger.debug(f"Found {self.Model.__name__}: {getattr(entity, 'id', None)}")
+            logger.trace(f"Found {self.Model.__name__}: {getattr(entity, 'id', None)}")
         else:
-            logger.debug(f"No {self.Model.__name__} found")
+            logger.trace(f"No {self.Model.__name__} found")
         return entity
 
     async def create(self, data: dict) -> T:
@@ -154,6 +203,15 @@ class Repository[T: Base]:
         async with db.scoped_session(self.session_maker) as session:
             # Only include valid columns that are provided in entity_data
             model_data = self.get_model_data(data)
+
+            # Add project_id if applicable and not already provided
+            if (
+                self.has_project_id
+                and self.project_id is not None
+                and "project_id" not in model_data
+            ):
+                model_data["project_id"] = self.project_id
+
             model = self.Model(**model_data)
             session.add(model)
             await session.flush()
@@ -176,12 +234,20 @@ class Repository[T: Base]:
 
         async with db.scoped_session(self.session_maker) as session:
             # Only include valid columns that are provided in entity_data
-            model_list = [
-                self.Model(
-                    **self.get_model_data(d),
-                )
-                for d in data_list
-            ]
+            model_list = []
+            for d in data_list:
+                model_data = self.get_model_data(d)
+
+                # Add project_id if applicable and not already provided
+                if (
+                    self.has_project_id
+                    and self.project_id is not None
+                    and "project_id" not in model_data
+                ):
+                    model_data["project_id"] = self.project_id  # pragma: no cover
+
+                model_list.append(self.Model(**model_data))
+
             session.add_all(model_list)
             await session.flush()
 
@@ -237,7 +303,13 @@ class Repository[T: Base]:
         """Delete records matching given IDs."""
         logger.debug(f"Deleting {self.Model.__name__} by ids: {ids}")
         async with db.scoped_session(self.session_maker) as session:
-            query = delete(self.Model).where(self.primary_key.in_(ids))
+            conditions = [self.primary_key.in_(ids)]
+
+            # Add project_id filter if applicable
+            if self.has_project_id and self.project_id is not None:  # pragma: no cover
+                conditions.append(getattr(self.Model, "project_id") == self.project_id)
+
+            query = delete(self.Model).where(and_(*conditions))
             result = await session.execute(query)
             logger.debug(f"Deleted {result.rowcount} records")
             return result.rowcount
@@ -247,6 +319,11 @@ class Repository[T: Base]:
         logger.debug(f"Deleting {self.Model.__name__} by fields: {filters}")
         async with db.scoped_session(self.session_maker) as session:
             conditions = [getattr(self.Model, field) == value for field, value in filters.items()]
+
+            # Add project_id filter if applicable
+            if self.has_project_id and self.project_id is not None:
+                conditions.append(getattr(self.Model, "project_id") == self.project_id)
+
             query = delete(self.Model).where(and_(*conditions))
             result = await session.execute(query)
             deleted = result.rowcount > 0
@@ -258,19 +335,34 @@ class Repository[T: Base]:
         async with db.scoped_session(self.session_maker) as session:
             if query is None:
                 query = select(func.count()).select_from(self.Model)
+                # Add project filter if applicable
+                if (
+                    isinstance(query, Select)
+                    and self.has_project_id
+                    and self.project_id is not None
+                ):
+                    query = query.where(
+                        getattr(self.Model, "project_id") == self.project_id
+                    )  # pragma: no cover
+
             result = await session.execute(query)
             scalar = result.scalar()
             count = scalar if scalar is not None else 0
             logger.debug(f"Counted {count} {self.Model.__name__} records")
             return count
 
-    async def execute_query(self, query: Executable, use_query_options: bool = True) -> Result[Any]:
+    async def execute_query(
+        self,
+        query: Executable,
+        params: Optional[Dict[str, Any]] = None,
+        use_query_options: bool = True,
+    ) -> Result[Any]:
         """Execute a query asynchronously."""
 
         query = query.options(*self.get_load_options()) if use_query_options else query
-        logger.debug(f"Executing query: {query}")
+        logger.trace(f"Executing query: {query}, params: {params}")
         async with db.scoped_session(self.session_maker) as session:
-            result = await session.execute(query)
+            result = await session.execute(query, params)
             return result
 
     def get_load_options(self) -> List[LoaderOption]:

basic_memory/repository/search_repository.py

@@ -19,6 +19,7 @@ from basic_memory.schemas.search import SearchItemType
 class SearchIndexRow:
     """Search result with score and metadata."""
 
+    project_id: int
     id: int
     type: str
     file_path: str
@@ -47,6 +48,27 @@ class SearchIndexRow:
     def content(self):
         return self.content_snippet
 
+    @property
+    def directory(self) -> str:
+        """Extract directory part from file_path.
+
+        For a file at "projects/notes/ideas.md", returns "/projects/notes"
+        For a file at root level "README.md", returns "/"
+        """
+        if not self.type == SearchItemType.ENTITY.value and not self.file_path:
+            return ""
+
+        # Split the path by slashes
+        parts = self.file_path.split("/")
+
+        # If there's only one part (e.g., "README.md"), it's at the root
+        if len(parts) <= 1:
+            return "/"
+
+        # Join all parts except the last one (filename)
+        directory_path = "/".join(parts[:-1])
+        return f"/{directory_path}"
+
     def to_insert(self):
         return {
             "id": self.id,
@@ -64,14 +86,28 @@ class SearchIndexRow:
             "category": self.category,
             "created_at": self.created_at if self.created_at else None,
             "updated_at": self.updated_at if self.updated_at else None,
+            "project_id": self.project_id,
         }
 
 
 class SearchRepository:
     """Repository for search index operations."""
 
-    def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
+    def __init__(self, session_maker: async_sessionmaker[AsyncSession], project_id: int):
+        """Initialize with session maker and project_id filter.
+
+        Args:
+            session_maker: SQLAlchemy session maker
+            project_id: Project ID to filter all operations by
+
+        Raises:
+            ValueError: If project_id is None or invalid
+        """
+        if project_id is None or project_id <= 0:  # pragma: no cover
+            raise ValueError("A valid project_id is required for SearchRepository")
+
         self.session_maker = session_maker
+        self.project_id = project_id
 
     async def init_search_index(self):
         """Create or recreate the search index."""
@@ -92,28 +128,93 @@ class SearchRepository:
             is_prefix: Whether to add prefix search capability (* suffix)
 
         For FTS5:
-        - Special characters and phrases need to be quoted
-        - Terms with spaces or special chars need quotes
-        - Boolean operators (AND, OR, NOT) and parentheses are preserved
+        - Boolean operators (AND, OR, NOT) are preserved for complex queries
+        - Terms with FTS5 special characters are quoted to prevent syntax errors
+        - Simple terms get prefix wildcards for better matching
         """
-        if "*" in term:
-            return term
-
-        # Check for boolean operators - if present, return the term as is
-        boolean_operators = [" AND ", " OR ", " NOT ", "(", ")"]
+        # Check for explicit boolean operators - if present, return the term as is
+        boolean_operators = [" AND ", " OR ", " NOT "]
         if any(op in f" {term} " for op in boolean_operators):
             return term
 
-        # List of special characters that need quoting (excluding *)
-        special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
+        # Check if term is already a proper wildcard pattern (alphanumeric + *)
+        # e.g., "hello*", "test*world" - these should be left alone
+        if "*" in term and all(c.isalnum() or c in "*_-" for c in term):
+            return term
 
-        # Check if term contains any special characters
-        needs_quotes = any(c in term for c in special_chars)
+        # Characters that can cause FTS5 syntax errors when used as operators
+        # We're more conservative here - only quote when we detect problematic patterns
+        problematic_chars = [
+            '"',
+            "'",
+            "(",
+            ")",
+            "[",
+            "]",
+            "{",
+            "}",
+            "+",
+            "!",
+            "@",
+            "#",
+            "$",
+            "%",
+            "^",
+            "&",
+            "=",
+            "|",
+            "\\",
+            "~",
+            "`",
+        ]
 
-        if needs_quotes:
-            # If the term already contains quotes, escape them and add a wildcard
-            term = term.replace('"', '""')
-            term = f'"{term}"*'
+        # Characters that indicate we should quote (spaces, dots, colons, etc.)
+        # Adding hyphens here because FTS5 can have issues with hyphens followed by wildcards
+        needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/", "-"]
+
+        # Check if term needs quoting
+        has_problematic = any(c in term for c in problematic_chars)
+        has_spaces_or_special = any(c in term for c in needs_quoting_chars)
+
+        if has_problematic or has_spaces_or_special:
+            # Handle multi-word queries differently from special character queries
+            if " " in term and not any(c in term for c in problematic_chars):
+                # Check if any individual word contains special characters that need quoting
+                words = term.strip().split()
+                has_special_in_words = any(
+                    any(c in word for c in needs_quoting_chars if c != " ") for word in words
+                )
+
+                if not has_special_in_words:
+                    # For multi-word queries with simple words (like "emoji unicode"),
+                    # use boolean AND to handle word order variations
+                    if is_prefix:
+                        # Add prefix wildcard to each word for better matching
+                        prepared_words = [f"{word}*" for word in words if word]
+                    else:
+                        prepared_words = words
+                    term = " AND ".join(prepared_words)
+                else:
+                    # If any word has special characters, quote the entire phrase
+                    escaped_term = term.replace('"', '""')
+                    if is_prefix and not ("/" in term and term.endswith(".md")):
+                        term = f'"{escaped_term}"*'
+                    else:
+                        term = f'"{escaped_term}"'
+            else:
+                # For terms with problematic characters or file paths, use exact phrase matching
+                # Escape any existing quotes by doubling them
+                escaped_term = term.replace('"', '""')
+                # Quote the entire term to handle special characters safely
+                if is_prefix and not ("/" in term and term.endswith(".md")):
+                    # For search terms (not file paths), add prefix matching
+                    term = f'"{escaped_term}"*'
+                else:
+                    # For file paths, use exact matching
+                    term = f'"{escaped_term}"'
+        elif is_prefix:
+            # Only add wildcard for simple terms without special characters
+            term = f"{term}*"
 
         return term
 
@@ -125,7 +226,7 @@ class SearchRepository:
         title: Optional[str] = None,
         types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
-        entity_types: Optional[List[SearchItemType]] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:
@@ -136,26 +237,30 @@ class SearchRepository:
 
         # Handle text search for title and content
         if search_text:
-            has_boolean = any(
-                op in f" {search_text} " for op in [" AND ", " OR ", " NOT ", "(", ")"]
-            )
-
-            if has_boolean:
-                # If boolean operators are present, use the raw query
-                # No need to prepare it, FTS5 will understand the operators
-                params["text"] = search_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+            # Skip FTS for wildcard-only queries that would cause "unknown special query" errors
+            if search_text.strip() == "*" or search_text.strip() == "":
+                # For wildcard searches, don't add any text conditions - return all results
+                pass
             else:
-                # Standard search with term preparation
-                processed_text = self._prepare_search_term(search_text.strip())
-                params["text"] = processed_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                # Check for explicit boolean operators - only detect them in proper boolean contexts
+                has_boolean = any(op in f" {search_text} " for op in [" AND ", " OR ", " NOT "])
+
+                if has_boolean:
+                    # If boolean operators are present, use the raw query
+                    # No need to prepare it, FTS5 will understand the operators
+                    params["text"] = search_text
+                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                else:
+                    # Standard search with term preparation
+                    processed_text = self._prepare_search_term(search_text.strip())
+                    params["text"] = processed_text
+                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
 
         # Handle title match search
         if title:
-            title_text = self._prepare_search_term(title.strip())
-            params["text"] = title_text
-            conditions.append("title MATCH :text")
+            title_text = self._prepare_search_term(title.strip(), is_prefix=False)
+            params["title_text"] = title_text
+            conditions.append("title MATCH :title_text")
 
         # Handle permalink exact search
         if permalink:
@@ -164,19 +269,25 @@ class SearchRepository:
 
         # Handle permalink match search, supports *
         if permalink_match:
-            # Clean and prepare permalink for FTS5 GLOB match
-            permalink_text = self._prepare_search_term(
-                permalink_match.lower().strip(), is_prefix=False
-            )
+            # For GLOB patterns, don't use _prepare_search_term as it will quote slashes
+            # GLOB patterns need to preserve their syntax
+            permalink_text = permalink_match.lower().strip()
             params["permalink"] = permalink_text
             if "*" in permalink_match:
                 conditions.append("permalink GLOB :permalink")
             else:
-                conditions.append("permalink MATCH :permalink")
+                # For exact matches without *, we can use FTS5 MATCH
+                # but only prepare the term if it doesn't look like a path
+                if "/" in permalink_text:
+                    conditions.append("permalink = :permalink")
+                else:
+                    permalink_text = self._prepare_search_term(permalink_text, is_prefix=False)
+                    params["permalink"] = permalink_text
+                    conditions.append("permalink MATCH :permalink")
 
         # Handle entity type filter
-        if entity_types:
-            type_list = ", ".join(f"'{t.value}'" for t in entity_types)
+        if search_item_types:
+            type_list = ", ".join(f"'{t.value}'" for t in search_item_types)
             conditions.append(f"type IN ({type_list})")
 
         # Handle type filter
@@ -192,6 +303,10 @@ class SearchRepository:
             # order by most recent first
             order_by_clause = ", updated_at DESC"
 
+        # Always filter by project_id
+        params["project_id"] = self.project_id
+        conditions.append("project_id = :project_id")
+
         # set limit on search query
         params["limit"] = limit
         params["offset"] = offset
@@ -201,6 +316,7 @@ class SearchRepository:
 
         sql = f"""
             SELECT 
+                project_id,
                 id, 
                 title, 
                 permalink,
@@ -224,12 +340,24 @@ class SearchRepository:
         """
 
         logger.trace(f"Search {sql} params: {params}")
-        async with db.scoped_session(self.session_maker) as session:
-            result = await session.execute(text(sql), params)
-            rows = result.fetchall()
+        try:
+            async with db.scoped_session(self.session_maker) as session:
+                result = await session.execute(text(sql), params)
+                rows = result.fetchall()
+        except Exception as e:
+            # Handle FTS5 syntax errors and provide user-friendly feedback
+            if "fts5: syntax error" in str(e).lower():  # pragma: no cover
+                logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
+                # Return empty results rather than crashing
+                return []
+            else:
+                # Re-raise other database errors
+                logger.error(f"Database error during search: {e}")
+                raise
 
         results = [
             SearchIndexRow(
+                project_id=self.project_id,
                 id=row.id,
                 title=row.title,
                 permalink=row.permalink,
@@ -249,10 +377,10 @@ class SearchRepository:
             for row in rows
         ]
 
-        logger.debug(f"Found {len(results)} search results")
+        logger.trace(f"Found {len(results)} search results")
         for r in results:
-            logger.debug(
-                f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}"
+            logger.trace(
+                f"Search result: project_id: {r.project_id} type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}"
             )
 
         return results
@@ -269,6 +397,10 @@ class SearchRepository:
                 {"permalink": search_index_row.permalink},
             )
 
+            # Prepare data for insert with project_id
+            insert_data = search_index_row.to_insert()
+            insert_data["project_id"] = self.project_id
+
             # Insert new record
             await session.execute(
                 text("""
@@ -276,15 +408,17 @@ class SearchRepository:
                         id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
                         from_id, to_id, relation_type,
                         entity_id, category,
-                        created_at, updated_at
+                        created_at, updated_at,
+                        project_id
                     ) VALUES (
                         :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
                         :from_id, :to_id, :relation_type,
                         :entity_id, :category,
-                        :created_at, :updated_at
+                        :created_at, :updated_at,
+                        :project_id
                     )
                 """),
-                search_index_row.to_insert(),
+                insert_data,
             )
             logger.debug(f"indexed row {search_index_row}")
             await session.commit()
@@ -293,8 +427,10 @@ class SearchRepository:
         """Delete an item from the search index by entity_id."""
         async with db.scoped_session(self.session_maker) as session:
             await session.execute(
-                text("DELETE FROM search_index WHERE entity_id = :entity_id"),
-                {"entity_id": entity_id},
+                text(
+                    "DELETE FROM search_index WHERE entity_id = :entity_id AND project_id = :project_id"
+                ),
+                {"entity_id": entity_id, "project_id": self.project_id},
             )
             await session.commit()
 
@@ -302,8 +438,10 @@ class SearchRepository:
         """Delete an item from the search index."""
         async with db.scoped_session(self.session_maker) as session:
             await session.execute(
-                text("DELETE FROM search_index WHERE permalink = :permalink"),
-                {"permalink": permalink},
+                text(
+                    "DELETE FROM search_index WHERE permalink = :permalink AND project_id = :project_id"
+                ),
+                {"permalink": permalink, "project_id": self.project_id},
             )
             await session.commit()
 

basic_memory/schemas/__init__.py

@@ -44,6 +44,10 @@ from basic_memory.schemas.project_info import (
     ProjectInfoResponse,
 )
 
+from basic_memory.schemas.directory import (
+    DirectoryNode,
+)
+
 # For convenient imports, export all models
 __all__ = [
     # Base
@@ -71,4 +75,6 @@ __all__ = [
     "ActivityMetrics",
     "SystemStatus",
     "ProjectInfoResponse",
+    # Directory
+    "DirectoryNode",
 ]