basic-memory 0.12.3__py3-none-any.whl → 0.13.0__py3-none-any.whl

basic_memory/repository/search_repository.py

@@ -19,6 +19,7 @@ from basic_memory.schemas.search import SearchItemType
 class SearchIndexRow:
     """Search result with score and metadata."""
 
+    project_id: int
     id: int
     type: str
     file_path: str
@@ -47,6 +48,27 @@ class SearchIndexRow:
     def content(self):
         return self.content_snippet
 
+    @property
+    def directory(self) -> str:
+        """Extract directory part from file_path.
+
+        For a file at "projects/notes/ideas.md", returns "/projects/notes"
+        For a file at root level "README.md", returns "/"
+        """
+        if not self.type == SearchItemType.ENTITY.value and not self.file_path:
+            return ""
+
+        # Split the path by slashes
+        parts = self.file_path.split("/")
+
+        # If there's only one part (e.g., "README.md"), it's at the root
+        if len(parts) <= 1:
+            return "/"
+
+        # Join all parts except the last one (filename)
+        directory_path = "/".join(parts[:-1])
+        return f"/{directory_path}"
+
     def to_insert(self):
         return {
             "id": self.id,
@@ -64,14 +86,28 @@ class SearchIndexRow:
             "category": self.category,
             "created_at": self.created_at if self.created_at else None,
             "updated_at": self.updated_at if self.updated_at else None,
+            "project_id": self.project_id,
         }
 
 
 class SearchRepository:
     """Repository for search index operations."""
 
-    def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
+    def __init__(self, session_maker: async_sessionmaker[AsyncSession], project_id: int):
+        """Initialize with session maker and project_id filter.
+
+        Args:
+            session_maker: SQLAlchemy session maker
+            project_id: Project ID to filter all operations by
+
+        Raises:
+            ValueError: If project_id is None or invalid
+        """
+        if project_id is None or project_id <= 0:  # pragma: no cover
+            raise ValueError("A valid project_id is required for SearchRepository")
+
         self.session_maker = session_maker
+        self.project_id = project_id
 
     async def init_search_index(self):
         """Create or recreate the search index."""
@@ -92,28 +128,93 @@ class SearchRepository:
             is_prefix: Whether to add prefix search capability (* suffix)
 
         For FTS5:
-        - Special characters and phrases need to be quoted
-        - Terms with spaces or special chars need quotes
-        - Boolean operators (AND, OR, NOT) and parentheses are preserved
+        - Boolean operators (AND, OR, NOT) are preserved for complex queries
+        - Terms with FTS5 special characters are quoted to prevent syntax errors
+        - Simple terms get prefix wildcards for better matching
         """
-        if "*" in term:
-            return term
-
-        # Check for boolean operators - if present, return the term as is
-        boolean_operators = [" AND ", " OR ", " NOT ", "(", ")"]
+        # Check for explicit boolean operators - if present, return the term as is
+        boolean_operators = [" AND ", " OR ", " NOT "]
         if any(op in f" {term} " for op in boolean_operators):
             return term
 
-        # List of special characters that need quoting (excluding *)
-        special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
+        # Check if term is already a proper wildcard pattern (alphanumeric + *)
+        # e.g., "hello*", "test*world" - these should be left alone
+        if "*" in term and all(c.isalnum() or c in "*_-" for c in term):
+            return term
 
-        # Check if term contains any special characters
-        needs_quotes = any(c in term for c in special_chars)
+        # Characters that can cause FTS5 syntax errors when used as operators
+        # We're more conservative here - only quote when we detect problematic patterns
+        problematic_chars = [
+            '"',
+            "'",
+            "(",
+            ")",
+            "[",
+            "]",
+            "{",
+            "}",
+            "+",
+            "!",
+            "@",
+            "#",
+            "$",
+            "%",
+            "^",
+            "&",
+            "=",
+            "|",
+            "\\",
+            "~",
+            "`",
+        ]
 
-        if needs_quotes:
-            # If the term already contains quotes, escape them and add a wildcard
-            term = term.replace('"', '""')
-            term = f'"{term}"*'
+        # Characters that indicate we should quote (spaces, dots, colons, etc.)
+        # Adding hyphens here because FTS5 can have issues with hyphens followed by wildcards
+        needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/", "-"]
+
+        # Check if term needs quoting
+        has_problematic = any(c in term for c in problematic_chars)
+        has_spaces_or_special = any(c in term for c in needs_quoting_chars)
+
+        if has_problematic or has_spaces_or_special:
+            # Handle multi-word queries differently from special character queries
+            if " " in term and not any(c in term for c in problematic_chars):
+                # Check if any individual word contains special characters that need quoting
+                words = term.strip().split()
+                has_special_in_words = any(
+                    any(c in word for c in needs_quoting_chars if c != " ") for word in words
+                )
+
+                if not has_special_in_words:
+                    # For multi-word queries with simple words (like "emoji unicode"),
+                    # use boolean AND to handle word order variations
+                    if is_prefix:
+                        # Add prefix wildcard to each word for better matching
+                        prepared_words = [f"{word}*" for word in words if word]
+                    else:
+                        prepared_words = words
+                    term = " AND ".join(prepared_words)
+                else:
+                    # If any word has special characters, quote the entire phrase
+                    escaped_term = term.replace('"', '""')
+                    if is_prefix and not ("/" in term and term.endswith(".md")):
+                        term = f'"{escaped_term}"*'
+                    else:
+                        term = f'"{escaped_term}"'
+            else:
+                # For terms with problematic characters or file paths, use exact phrase matching
+                # Escape any existing quotes by doubling them
+                escaped_term = term.replace('"', '""')
+                # Quote the entire term to handle special characters safely
+                if is_prefix and not ("/" in term and term.endswith(".md")):
+                    # For search terms (not file paths), add prefix matching
+                    term = f'"{escaped_term}"*'
+                else:
+                    # For file paths, use exact matching
+                    term = f'"{escaped_term}"'
+        elif is_prefix:
+            # Only add wildcard for simple terms without special characters
+            term = f"{term}*"
 
         return term
 
@@ -125,7 +226,7 @@ class SearchRepository:
         title: Optional[str] = None,
         types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
-        entity_types: Optional[List[SearchItemType]] = None,
+        search_item_types: Optional[List[SearchItemType]] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:
@@ -136,26 +237,30 @@ class SearchRepository:
 
         # Handle text search for title and content
         if search_text:
-            has_boolean = any(
-                op in f" {search_text} " for op in [" AND ", " OR ", " NOT ", "(", ")"]
-            )
-
-            if has_boolean:
-                # If boolean operators are present, use the raw query
-                # No need to prepare it, FTS5 will understand the operators
-                params["text"] = search_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+            # Skip FTS for wildcard-only queries that would cause "unknown special query" errors
+            if search_text.strip() == "*" or search_text.strip() == "":
+                # For wildcard searches, don't add any text conditions - return all results
+                pass
             else:
-                # Standard search with term preparation
-                processed_text = self._prepare_search_term(search_text.strip())
-                params["text"] = processed_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                # Check for explicit boolean operators - only detect them in proper boolean contexts
+                has_boolean = any(op in f" {search_text} " for op in [" AND ", " OR ", " NOT "])
+
+                if has_boolean:
+                    # If boolean operators are present, use the raw query
+                    # No need to prepare it, FTS5 will understand the operators
+                    params["text"] = search_text
+                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                else:
+                    # Standard search with term preparation
+                    processed_text = self._prepare_search_term(search_text.strip())
+                    params["text"] = processed_text
+                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
 
         # Handle title match search
         if title:
-            title_text = self._prepare_search_term(title.strip())
-            params["text"] = title_text
-            conditions.append("title MATCH :text")
+            title_text = self._prepare_search_term(title.strip(), is_prefix=False)
+            params["title_text"] = title_text
+            conditions.append("title MATCH :title_text")
 
         # Handle permalink exact search
         if permalink:
@@ -164,19 +269,25 @@ class SearchRepository:
 
         # Handle permalink match search, supports *
         if permalink_match:
-            # Clean and prepare permalink for FTS5 GLOB match
-            permalink_text = self._prepare_search_term(
-                permalink_match.lower().strip(), is_prefix=False
-            )
+            # For GLOB patterns, don't use _prepare_search_term as it will quote slashes
+            # GLOB patterns need to preserve their syntax
+            permalink_text = permalink_match.lower().strip()
             params["permalink"] = permalink_text
             if "*" in permalink_match:
                 conditions.append("permalink GLOB :permalink")
             else:
-                conditions.append("permalink MATCH :permalink")
+                # For exact matches without *, we can use FTS5 MATCH
+                # but only prepare the term if it doesn't look like a path
+                if "/" in permalink_text:
+                    conditions.append("permalink = :permalink")
+                else:
+                    permalink_text = self._prepare_search_term(permalink_text, is_prefix=False)
+                    params["permalink"] = permalink_text
+                    conditions.append("permalink MATCH :permalink")
 
         # Handle entity type filter
-        if entity_types:
-            type_list = ", ".join(f"'{t.value}'" for t in entity_types)
+        if search_item_types:
+            type_list = ", ".join(f"'{t.value}'" for t in search_item_types)
             conditions.append(f"type IN ({type_list})")
 
         # Handle type filter
@@ -192,6 +303,10 @@ class SearchRepository:
             # order by most recent first
             order_by_clause = ", updated_at DESC"
 
+        # Always filter by project_id
+        params["project_id"] = self.project_id
+        conditions.append("project_id = :project_id")
+
         # set limit on search query
         params["limit"] = limit
         params["offset"] = offset
@@ -201,6 +316,7 @@ class SearchRepository:
 
         sql = f"""
             SELECT 
+                project_id,
                 id, 
                 title, 
                 permalink,
@@ -224,12 +340,24 @@ class SearchRepository:
         """
 
         logger.trace(f"Search {sql} params: {params}")
-        async with db.scoped_session(self.session_maker) as session:
-            result = await session.execute(text(sql), params)
-            rows = result.fetchall()
+        try:
+            async with db.scoped_session(self.session_maker) as session:
+                result = await session.execute(text(sql), params)
+                rows = result.fetchall()
+        except Exception as e:
+            # Handle FTS5 syntax errors and provide user-friendly feedback
+            if "fts5: syntax error" in str(e).lower():  # pragma: no cover
+                logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
+                # Return empty results rather than crashing
+                return []
+            else:
+                # Re-raise other database errors
+                logger.error(f"Database error during search: {e}")
+                raise
 
         results = [
             SearchIndexRow(
+                project_id=self.project_id,
                 id=row.id,
                 title=row.title,
                 permalink=row.permalink,
@@ -249,10 +377,10 @@ class SearchRepository:
             for row in rows
         ]
 
-        logger.debug(f"Found {len(results)} search results")
+        logger.trace(f"Found {len(results)} search results")
         for r in results:
-            logger.debug(
-                f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}"
+            logger.trace(
+                f"Search result: project_id: {r.project_id} type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}"
             )
 
         return results
@@ -269,6 +397,10 @@ class SearchRepository:
                 {"permalink": search_index_row.permalink},
             )
 
+            # Prepare data for insert with project_id
+            insert_data = search_index_row.to_insert()
+            insert_data["project_id"] = self.project_id
+
             # Insert new record
             await session.execute(
                 text("""
@@ -276,15 +408,17 @@ class SearchRepository:
                         id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
                         from_id, to_id, relation_type,
                         entity_id, category,
-                        created_at, updated_at
+                        created_at, updated_at,
+                        project_id
                     ) VALUES (
                         :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
                         :from_id, :to_id, :relation_type,
                         :entity_id, :category,
-                        :created_at, :updated_at
+                        :created_at, :updated_at,
+                        :project_id
                     )
                 """),
-                search_index_row.to_insert(),
+                insert_data,
             )
             logger.debug(f"indexed row {search_index_row}")
             await session.commit()
@@ -293,8 +427,10 @@ class SearchRepository:
         """Delete an item from the search index by entity_id."""
         async with db.scoped_session(self.session_maker) as session:
             await session.execute(
-                text("DELETE FROM search_index WHERE entity_id = :entity_id"),
-                {"entity_id": entity_id},
+                text(
+                    "DELETE FROM search_index WHERE entity_id = :entity_id AND project_id = :project_id"
+                ),
+                {"entity_id": entity_id, "project_id": self.project_id},
             )
             await session.commit()
 
@@ -302,8 +438,10 @@ class SearchRepository:
         """Delete an item from the search index."""
         async with db.scoped_session(self.session_maker) as session:
             await session.execute(
-                text("DELETE FROM search_index WHERE permalink = :permalink"),
-                {"permalink": permalink},
+                text(
+                    "DELETE FROM search_index WHERE permalink = :permalink AND project_id = :project_id"
+                ),
+                {"permalink": permalink, "project_id": self.project_id},
             )
             await session.commit()
 

basic_memory/schemas/__init__.py

@@ -44,6 +44,10 @@ from basic_memory.schemas.project_info import (
     ProjectInfoResponse,
 )
 
+from basic_memory.schemas.directory import (
+    DirectoryNode,
+)
+
 # For convenient imports, export all models
 __all__ = [
     # Base
@@ -71,4 +75,6 @@ __all__ = [
     "ActivityMetrics",
     "SystemStatus",
     "ProjectInfoResponse",
+    # Directory
+    "DirectoryNode",
 ]

basic_memory/schemas/base.py

@@ -13,7 +13,7 @@ Key Concepts:
 
 import mimetypes
 import re
-from datetime import datetime
+from datetime import datetime, time
 from pathlib import Path
 from typing import List, Optional, Annotated, Dict
 
@@ -46,15 +46,43 @@ def to_snake_case(name: str) -> str:
     return s2.lower()
 
 
+def parse_timeframe(timeframe: str) -> datetime:
+    """Parse timeframe with special handling for 'today' and other natural language expressions.
+
+    Args:
+        timeframe: Natural language timeframe like 'today', '1d', '1 week ago', etc.
+
+    Returns:
+        datetime: The parsed datetime for the start of the timeframe
+
+    Examples:
+        parse_timeframe('today') -> 2025-06-05 00:00:00 (start of today)
+        parse_timeframe('1d') -> 2025-06-04 14:50:00 (24 hours ago)
+        parse_timeframe('1 week ago') -> 2025-05-29 14:50:00 (1 week ago)
+    """
+    if timeframe.lower() == "today":
+        # Return start of today (00:00:00)
+        return datetime.combine(datetime.now().date(), time.min)
+    else:
+        # Use dateparser for other formats
+        parsed = parse(timeframe)
+        if not parsed:
+            raise ValueError(f"Could not parse timeframe: {timeframe}")
+        return parsed
+
+
 def validate_timeframe(timeframe: str) -> str:
     """Convert human readable timeframes to a duration relative to the current time."""
     if not isinstance(timeframe, str):
         raise ValueError("Timeframe must be a string")
 
-    # Parse relative time expression
-    parsed = parse(timeframe)
-    if not parsed:
-        raise ValueError(f"Could not parse timeframe: {timeframe}")
+    # Preserve special timeframe strings that need custom handling
+    special_timeframes = ["today"]
+    if timeframe.lower() in special_timeframes:
+        return timeframe.lower()
+
+    # Parse relative time expression using our enhanced parser
+    parsed = parse_timeframe(timeframe)
 
     # Convert to duration
     now = datetime.now()

basic_memory/schemas/directory.py

@@ -0,0 +1,30 @@
+"""Schemas for directory tree operations."""
+
+from datetime import datetime
+from typing import List, Optional, Literal
+
+from pydantic import BaseModel
+
+
+class DirectoryNode(BaseModel):
+    """Directory node in file system."""
+
+    name: str
+    file_path: Optional[str] = None  # Original path without leading slash (matches DB)
+    directory_path: str  # Path with leading slash for directory navigation
+    type: Literal["directory", "file"]
+    children: List["DirectoryNode"] = []  # Default to empty list
+    title: Optional[str] = None
+    permalink: Optional[str] = None
+    entity_id: Optional[int] = None
+    entity_type: Optional[str] = None
+    content_type: Optional[str] = None
+    updated_at: Optional[datetime] = None
+
+    @property
+    def has_children(self) -> bool:
+        return bool(self.children)
+
+
+# Support for recursive model
+DirectoryNode.model_rebuild()

basic_memory/schemas/importer.py

@@ -0,0 +1,34 @@
+"""Schemas for import services."""
+
+from typing import Dict, Optional
+
+from pydantic import BaseModel
+
+
+class ImportResult(BaseModel):
+    """Common import result schema."""
+
+    import_count: Dict[str, int]
+    success: bool
+    error_message: Optional[str] = None
+
+
+class ChatImportResult(ImportResult):
+    """Result schema for chat imports."""
+
+    conversations: int = 0
+    messages: int = 0
+
+
+class ProjectImportResult(ImportResult):
+    """Result schema for project imports."""
+
+    documents: int = 0
+    prompts: int = 0
+
+
+class EntityImportResult(ImportResult):
+    """Result schema for entity imports."""
+
+    entities: int = 0
+    relations: int = 0

basic_memory/schemas/memory.py

@@ -9,8 +9,44 @@ from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
 from basic_memory.schemas.search import SearchItemType
 
 
+def validate_memory_url_path(path: str) -> bool:
+    """Validate that a memory URL path is well-formed.
+
+    Args:
+        path: The path part of a memory URL (without memory:// prefix)
+
+    Returns:
+        True if the path is valid, False otherwise
+
+    Examples:
+        >>> validate_memory_url_path("specs/search")
+        True
+        >>> validate_memory_url_path("memory//test")  # Double slash
+        False
+        >>> validate_memory_url_path("invalid://test")  # Contains protocol
+        False
+    """
+    if not path or not path.strip():
+        return False
+
+    # Check for invalid protocol schemes within the path first (more specific)
+    if "://" in path:
+        return False
+
+    # Check for double slashes (except at the beginning for absolute paths)
+    if "//" in path:
+        return False
+
+    # Check for invalid characters (excluding * which is used for pattern matching)
+    invalid_chars = {"<", ">", '"', "|", "?"}
+    if any(char in path for char in invalid_chars):
+        return False
+
+    return True
+
+
 def normalize_memory_url(url: str | None) -> str:
-    """Normalize a MemoryUrl string.
+    """Normalize a MemoryUrl string with validation.
 
     Args:
         url: A path like "specs/search" or "memory://specs/search"
@@ -18,22 +54,43 @@ def normalize_memory_url(url: str | None) -> str:
     Returns:
         Normalized URL starting with memory://
 
+    Raises:
+        ValueError: If the URL path is malformed
+
     Examples:
         >>> normalize_memory_url("specs/search")
         'memory://specs/search'
         >>> normalize_memory_url("memory://specs/search")
         'memory://specs/search'
+        >>> normalize_memory_url("memory//test")
+        Traceback (most recent call last):
+        ...
+        ValueError: Invalid memory URL path: 'memory//test' contains double slashes
     """
     if not url:
         return ""
 
     clean_path = url.removeprefix("memory://")
+
+    # Validate the extracted path
+    if not validate_memory_url_path(clean_path):
+        # Provide specific error messages for common issues
+        if "://" in clean_path:
+            raise ValueError(f"Invalid memory URL path: '{clean_path}' contains protocol scheme")
+        elif "//" in clean_path:
+            raise ValueError(f"Invalid memory URL path: '{clean_path}' contains double slashes")
+        elif not clean_path.strip():
+            raise ValueError("Memory URL path cannot be empty or whitespace")
+        else:
+            raise ValueError(f"Invalid memory URL path: '{clean_path}' contains invalid characters")
+
     return f"memory://{clean_path}"
 
 
 MemoryUrl = Annotated[
     str,
     BeforeValidator(str.strip),  # Clean whitespace
+    BeforeValidator(normalize_memory_url),  # Validate and normalize the URL
     MinLen(1),
     MaxLen(2028),
 ]
@@ -100,27 +157,41 @@ class MemoryMetadata(BaseModel):
     uri: Optional[str] = None
     types: Optional[List[SearchItemType]] = None
     depth: int
-    timeframe: str
+    timeframe: Optional[str] = None
     generated_at: datetime
-    total_results: int
-    total_relations: int
+    primary_count: Optional[int] = None  # Changed field name
+    related_count: Optional[int] = None  # Changed field name
+    total_results: Optional[int] = None  # For backward compatibility
+    total_relations: Optional[int] = None
+    total_observations: Optional[int] = None
 
 
-class GraphContext(BaseModel):
-    """Complete context response."""
+class ContextResult(BaseModel):
+    """Context result containing a primary item with its observations and related items."""
+
+    primary_result: EntitySummary | RelationSummary | ObservationSummary = Field(
+        description="Primary item"
+    )
 
-    # Direct matches
-    primary_results: Sequence[EntitySummary | RelationSummary | ObservationSummary] = Field(
-        description="results directly matching URI"
+    observations: Sequence[ObservationSummary] = Field(
+        description="Observations belonging to this entity", default_factory=list
     )
 
-    # Related entities
     related_results: Sequence[EntitySummary | RelationSummary | ObservationSummary] = Field(
-        description="related results"
+        description="Related items", default_factory=list
+    )
+
+
+class GraphContext(BaseModel):
+    """Complete context response."""
+
+    # hierarchical results
+    results: Sequence[ContextResult] = Field(
+        description="Hierarchical results with related items nested", default_factory=list
     )
 
     # Context metadata
     metadata: MemoryMetadata
 
-    page: int = 1
-    page_size: int = 1
+    page: Optional[int] = None
+    page_size: Optional[int] = None