basic-memory 0.13.0b4__py3-none-any.whl → 0.13.0b6__py3-none-any.whl

basic_memory/repository/search_repository.py

@@ -128,34 +128,90 @@ class SearchRepository:
             is_prefix: Whether to add prefix search capability (* suffix)
 
         For FTS5:
-        - Special characters and phrases need to be quoted
-        - Terms with spaces or special chars need quotes
         - Boolean operators (AND, OR, NOT) are preserved for complex queries
+        - Terms with FTS5 special characters are quoted to prevent syntax errors
+        - Simple terms get prefix wildcards for better matching
         """
-        if "*" in term:
-            return term
-
         # Check for explicit boolean operators - if present, return the term as is
         boolean_operators = [" AND ", " OR ", " NOT "]
         if any(op in f" {term} " for op in boolean_operators):
             return term
 
-        # List of FTS5 special characters that need escaping/quoting
-        special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
+        # Check if term is already a proper wildcard pattern (alphanumeric + *)
+        # e.g., "hello*", "test*world" - these should be left alone
+        if "*" in term and all(c.isalnum() or c in "*_-" for c in term):
+            return term
 
-        # Check if term contains any special characters
-        needs_quotes = any(c in term for c in special_chars)
+        # Characters that can cause FTS5 syntax errors when used as operators
+        # We're more conservative here - only quote when we detect problematic patterns
+        problematic_chars = [
+            '"',
+            "'",
+            "(",
+            ")",
+            "[",
+            "]",
+            "{",
+            "}",
+            "+",
+            "!",
+            "@",
+            "#",
+            "$",
+            "%",
+            "^",
+            "&",
+            "=",
+            "|",
+            "\\",
+            "~",
+            "`",
+        ]
 
-        if needs_quotes:
-            # Escape any existing quotes by doubling them
-            escaped_term = term.replace('"', '""')
-            # Quote the entire term to handle special characters safely
-            if is_prefix and not ("/" in term and term.endswith(".md")):
-                # For search terms (not file paths), add prefix matching
-                term = f'"{escaped_term}"*'
+        # Characters that indicate we should quote (spaces, dots, colons, etc.)
+        # Adding hyphens here because FTS5 can have issues with hyphens followed by wildcards
+        needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/", "-"]
+
+        # Check if term needs quoting
+        has_problematic = any(c in term for c in problematic_chars)
+        has_spaces_or_special = any(c in term for c in needs_quoting_chars)
+
+        if has_problematic or has_spaces_or_special:
+            # Handle multi-word queries differently from special character queries
+            if " " in term and not any(c in term for c in problematic_chars):
+                # Check if any individual word contains special characters that need quoting
+                words = term.strip().split()
+                has_special_in_words = any(
+                    any(c in word for c in needs_quoting_chars if c != " ") for word in words
+                )
+
+                if not has_special_in_words:
+                    # For multi-word queries with simple words (like "emoji unicode"),
+                    # use boolean AND to handle word order variations
+                    if is_prefix:
+                        # Add prefix wildcard to each word for better matching
+                        prepared_words = [f"{word}*" for word in words if word]
+                    else:
+                        prepared_words = words
+                    term = " AND ".join(prepared_words)
+                else:
+                    # If any word has special characters, quote the entire phrase
+                    escaped_term = term.replace('"', '""')
+                    if is_prefix and not ("/" in term and term.endswith(".md")):
+                        term = f'"{escaped_term}"*'
+                    else:
+                        term = f'"{escaped_term}"'
             else:
-                # For file paths, use exact matching
-                term = f'"{escaped_term}"'
+                # For terms with problematic characters or file paths, use exact phrase matching
+                # Escape any existing quotes by doubling them
+                escaped_term = term.replace('"', '""')
+                # Quote the entire term to handle special characters safely
+                if is_prefix and not ("/" in term and term.endswith(".md")):
+                    # For search terms (not file paths), add prefix matching
+                    term = f'"{escaped_term}"*'
+                else:
+                    # For file paths, use exact matching
+                    term = f'"{escaped_term}"'
         elif is_prefix:
             # Only add wildcard for simple terms without special characters
             term = f"{term}*"
@@ -181,19 +237,24 @@ class SearchRepository:
 
         # Handle text search for title and content
         if search_text:
-            # Check for explicit boolean operators - only detect them in proper boolean contexts
-            has_boolean = any(op in f" {search_text} " for op in [" AND ", " OR ", " NOT "])
-
-            if has_boolean:
-                # If boolean operators are present, use the raw query
-                # No need to prepare it, FTS5 will understand the operators
-                params["text"] = search_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+            # Skip FTS for wildcard-only queries that would cause "unknown special query" errors
+            if search_text.strip() == "*" or search_text.strip() == "":
+                # For wildcard searches, don't add any text conditions - return all results
+                pass
             else:
-                # Standard search with term preparation
-                processed_text = self._prepare_search_term(search_text.strip())
-                params["text"] = processed_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                # Check for explicit boolean operators - only detect them in proper boolean contexts
+                has_boolean = any(op in f" {search_text} " for op in [" AND ", " OR ", " NOT "])
+
+                if has_boolean:
+                    # If boolean operators are present, use the raw query
+                    # No need to prepare it, FTS5 will understand the operators
+                    params["text"] = search_text
+                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                else:
+                    # Standard search with term preparation
+                    processed_text = self._prepare_search_term(search_text.strip())
+                    params["text"] = processed_text
+                    conditions.append("(title MATCH :text OR content_stems MATCH :text)")
 
         # Handle title match search
         if title:
@@ -208,15 +269,21 @@ class SearchRepository:
 
         # Handle permalink match search, supports *
         if permalink_match:
-            # Clean and prepare permalink for FTS5 GLOB match
-            permalink_text = self._prepare_search_term(
-                permalink_match.lower().strip(), is_prefix=False
-            )
+            # For GLOB patterns, don't use _prepare_search_term as it will quote slashes
+            # GLOB patterns need to preserve their syntax
+            permalink_text = permalink_match.lower().strip()
             params["permalink"] = permalink_text
             if "*" in permalink_match:
                 conditions.append("permalink GLOB :permalink")
             else:
-                conditions.append("permalink MATCH :permalink")
+                # For exact matches without *, we can use FTS5 MATCH
+                # but only prepare the term if it doesn't look like a path
+                if "/" in permalink_text:
+                    conditions.append("permalink = :permalink")
+                else:
+                    permalink_text = self._prepare_search_term(permalink_text, is_prefix=False)
+                    params["permalink"] = permalink_text
+                    conditions.append("permalink MATCH :permalink")
 
         # Handle entity type filter
         if search_item_types:
@@ -273,9 +340,20 @@ class SearchRepository:
         """
 
         logger.trace(f"Search {sql} params: {params}")
-        async with db.scoped_session(self.session_maker) as session:
-            result = await session.execute(text(sql), params)
-            rows = result.fetchall()
+        try:
+            async with db.scoped_session(self.session_maker) as session:
+                result = await session.execute(text(sql), params)
+                rows = result.fetchall()
+        except Exception as e:
+            # Handle FTS5 syntax errors and provide user-friendly feedback
+            if "fts5: syntax error" in str(e).lower():  # pragma: no cover
+                logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
+                # Return empty results rather than crashing
+                return []
+            else:
+                # Re-raise other database errors
+                logger.error(f"Database error during search: {e}")
+                raise
 
         results = [
             SearchIndexRow(

basic_memory/schemas/base.py

@@ -13,7 +13,7 @@ Key Concepts:
 
 import mimetypes
 import re
-from datetime import datetime
+from datetime import datetime, time
 from pathlib import Path
 from typing import List, Optional, Annotated, Dict
 
@@ -46,15 +46,43 @@ def to_snake_case(name: str) -> str:
     return s2.lower()
 
 
+def parse_timeframe(timeframe: str) -> datetime:
+    """Parse timeframe with special handling for 'today' and other natural language expressions.
+
+    Args:
+        timeframe: Natural language timeframe like 'today', '1d', '1 week ago', etc.
+
+    Returns:
+        datetime: The parsed datetime for the start of the timeframe
+
+    Examples:
+        parse_timeframe('today') -> 2025-06-05 00:00:00 (start of today)
+        parse_timeframe('1d') -> 2025-06-04 14:50:00 (24 hours ago)
+        parse_timeframe('1 week ago') -> 2025-05-29 14:50:00 (1 week ago)
+    """
+    if timeframe.lower() == "today":
+        # Return start of today (00:00:00)
+        return datetime.combine(datetime.now().date(), time.min)
+    else:
+        # Use dateparser for other formats
+        parsed = parse(timeframe)
+        if not parsed:
+            raise ValueError(f"Could not parse timeframe: {timeframe}")
+        return parsed
+
+
 def validate_timeframe(timeframe: str) -> str:
     """Convert human readable timeframes to a duration relative to the current time."""
     if not isinstance(timeframe, str):
         raise ValueError("Timeframe must be a string")
 
-    # Parse relative time expression
-    parsed = parse(timeframe)
-    if not parsed:
-        raise ValueError(f"Could not parse timeframe: {timeframe}")
+    # Preserve special timeframe strings that need custom handling
+    special_timeframes = ["today"]
+    if timeframe.lower() in special_timeframes:
+        return timeframe.lower()
+
+    # Parse relative time expression using our enhanced parser
+    parsed = parse_timeframe(timeframe)
 
     # Convert to duration
     now = datetime.now()

basic_memory/schemas/memory.py

@@ -9,8 +9,44 @@ from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
 from basic_memory.schemas.search import SearchItemType
 
 
+def validate_memory_url_path(path: str) -> bool:
+    """Validate that a memory URL path is well-formed.
+
+    Args:
+        path: The path part of a memory URL (without memory:// prefix)
+
+    Returns:
+        True if the path is valid, False otherwise
+
+    Examples:
+        >>> validate_memory_url_path("specs/search")
+        True
+        >>> validate_memory_url_path("memory//test")  # Double slash
+        False
+        >>> validate_memory_url_path("invalid://test")  # Contains protocol
+        False
+    """
+    if not path or not path.strip():
+        return False
+
+    # Check for invalid protocol schemes within the path first (more specific)
+    if "://" in path:
+        return False
+
+    # Check for double slashes (except at the beginning for absolute paths)
+    if "//" in path:
+        return False
+
+    # Check for invalid characters (excluding * which is used for pattern matching)
+    invalid_chars = {"<", ">", '"', "|", "?"}
+    if any(char in path for char in invalid_chars):
+        return False
+
+    return True
+
+
 def normalize_memory_url(url: str | None) -> str:
-    """Normalize a MemoryUrl string.
+    """Normalize a MemoryUrl string with validation.
 
     Args:
         url: A path like "specs/search" or "memory://specs/search"
@@ -18,22 +54,43 @@ def normalize_memory_url(url: str | None) -> str:
     Returns:
         Normalized URL starting with memory://
 
+    Raises:
+        ValueError: If the URL path is malformed
+
     Examples:
         >>> normalize_memory_url("specs/search")
         'memory://specs/search'
         >>> normalize_memory_url("memory://specs/search")
         'memory://specs/search'
+        >>> normalize_memory_url("memory//test")
+        Traceback (most recent call last):
+        ...
+        ValueError: Invalid memory URL path: 'memory//test' contains double slashes
     """
     if not url:
         return ""
 
     clean_path = url.removeprefix("memory://")
+
+    # Validate the extracted path
+    if not validate_memory_url_path(clean_path):
+        # Provide specific error messages for common issues
+        if "://" in clean_path:
+            raise ValueError(f"Invalid memory URL path: '{clean_path}' contains protocol scheme")
+        elif "//" in clean_path:
+            raise ValueError(f"Invalid memory URL path: '{clean_path}' contains double slashes")
+        elif not clean_path.strip():
+            raise ValueError("Memory URL path cannot be empty or whitespace")
+        else:
+            raise ValueError(f"Invalid memory URL path: '{clean_path}' contains invalid characters")
+
     return f"memory://{clean_path}"
 
 
 MemoryUrl = Annotated[
     str,
     BeforeValidator(str.strip),  # Clean whitespace
+    BeforeValidator(normalize_memory_url),  # Validate and normalize the URL
     MinLen(1),
     MaxLen(2028),
 ]

basic_memory/services/entity_service.py

@@ -299,7 +299,20 @@ class EntityService(BaseService[EntityModel]):
         # Mark as incomplete because we still need to add relations
         model.checksum = None
         # Repository will set project_id automatically
-        return await self.repository.add(model)
+        try:
+            return await self.repository.add(model)
+        except IntegrityError as e:
+            # Handle race condition where entity was created by another process
+            if "UNIQUE constraint failed: entity.file_path" in str(
+                e
+            ) or "UNIQUE constraint failed: entity.permalink" in str(e):
+                logger.info(
+                    f"Entity already exists for file_path={file_path} (file_path or permalink conflict), updating instead of creating"
+                )
+                return await self.update_entity_and_observations(file_path, markdown)
+            else:
+                # Re-raise if it's a different integrity error
+                raise
 
     async def update_entity_and_observations(
         self, file_path: Path, markdown: EntityMarkdown
@@ -413,8 +426,8 @@ class EntityService(BaseService[EntityModel]):
         """
         logger.debug(f"Editing entity: {identifier}, operation: {operation}")
 
-        # Find the entity using the link resolver
-        entity = await self.link_resolver.resolve_link(identifier)
+        # Find the entity using the link resolver with strict mode for destructive operations
+        entity = await self.link_resolver.resolve_link(identifier, strict=True)
         if not entity:
             raise EntityNotFoundError(f"Entity not found: {identifier}")
 
@@ -630,8 +643,8 @@ class EntityService(BaseService[EntityModel]):
         """
         logger.debug(f"Moving entity: {identifier} to {destination_path}")
 
-        # 1. Resolve identifier to entity
-        entity = await self.link_resolver.resolve_link(identifier)
+        # 1. Resolve identifier to entity with strict mode for destructive operations
+        entity = await self.link_resolver.resolve_link(identifier, strict=True)
         if not entity:
             raise EntityNotFoundError(f"Entity not found: {identifier}")
 

basic_memory/services/initialization.py

@@ -83,7 +83,9 @@ async def migrate_legacy_projects(app_config: BasicMemoryConfig):
             logger.error(f"Project {project_name} not found in database, skipping migration")
             continue
 
+        logger.info(f"Starting migration for project: {project_name} (id: {project.id})")
         await migrate_legacy_project_data(project, legacy_dir)
+        logger.info(f"Completed migration for project: {project_name}")
     logger.info("Legacy projects successfully migrated")
 
 
@@ -104,7 +106,7 @@ async def migrate_legacy_project_data(project: Project, legacy_dir: Path) -> boo
     sync_dir = Path(project.path)
 
     logger.info(f"Sync starting project: {project.name}")
-    await sync_service.sync(sync_dir)
+    await sync_service.sync(sync_dir, project_name=project.name)
     logger.info(f"Sync completed successfully for project: {project.name}")
 
     # After successful sync, remove the legacy directory
@@ -158,12 +160,32 @@ async def initialize_file_sync(
         sync_dir = Path(project.path)
 
         try:
-            await sync_service.sync(sync_dir)
+            await sync_service.sync(sync_dir, project_name=project.name)
             logger.info(f"Sync completed successfully for project: {project.name}")
+
+            # Mark project as watching for changes after successful sync
+            from basic_memory.services.sync_status_service import sync_status_tracker
+
+            sync_status_tracker.start_project_watch(project.name)
+            logger.info(f"Project {project.name} is now watching for changes")
         except Exception as e:  # pragma: no cover
             logger.error(f"Error syncing project {project.name}: {e}")
+            # Mark sync as failed for this project
+            from basic_memory.services.sync_status_service import sync_status_tracker
+
+            sync_status_tracker.fail_project_sync(project.name, str(e))
             # Continue with other projects even if one fails
 
+    # Mark migration complete if it was in progress
+    try:
+        from basic_memory.services.migration_service import migration_manager
+
+        if not migration_manager.is_ready:  # pragma: no cover
+            migration_manager.mark_completed("Migration completed with file sync")
+            logger.info("Marked migration as completed after file sync")
+    except Exception as e:  # pragma: no cover
+        logger.warning(f"Could not update migration status: {e}")
+
     # Then start the watch service in the background
     logger.info("Starting watch service for all projects")
     # run the watch service
@@ -185,7 +207,7 @@ async def initialize_app(
     - Running database migrations
     - Reconciling projects from config.json with projects table
     - Setting up file synchronization
-    - Migrating legacy project data
+    - Starting background migration for legacy project data
 
     Args:
         app_config: The Basic Memory project configuration
@@ -197,8 +219,13 @@ async def initialize_app(
     # Reconcile projects from config.json with projects table
     await reconcile_projects_with_config(app_config)
 
-    # migrate legacy project data
-    await migrate_legacy_projects(app_config)
+    # Start background migration for legacy project data (non-blocking)
+    from basic_memory.services.migration_service import migration_manager
+
+    await migration_manager.start_background_migration(app_config)
+
+    logger.info("App initialization completed (migration running in background if needed)")
+    return migration_manager
 
 
 def ensure_initialization(app_config: BasicMemoryConfig) -> None:

basic_memory/services/link_resolver.py

@@ -26,8 +26,16 @@ class LinkResolver:
         self.entity_repository = entity_repository
         self.search_service = search_service
 
-    async def resolve_link(self, link_text: str, use_search: bool = True) -> Optional[Entity]:
-        """Resolve a markdown link to a permalink."""
+    async def resolve_link(
+        self, link_text: str, use_search: bool = True, strict: bool = False
+    ) -> Optional[Entity]:
+        """Resolve a markdown link to a permalink.
+
+        Args:
+            link_text: The link text to resolve
+            use_search: Whether to use search-based fuzzy matching as fallback
+            strict: If True, only exact matches are allowed (no fuzzy search fallback)
+        """
         logger.trace(f"Resolving link: {link_text}")
 
         # Clean link text and extract any alias
@@ -41,7 +49,8 @@ class LinkResolver:
 
         # 2. Try exact title match
         found = await self.entity_repository.get_by_title(clean_text)
-        if found and len(found) == 1:
+        if found:
+            # Return first match if there are duplicates (consistent behavior)
             entity = found[0]
             logger.debug(f"Found title match: {entity.title}")
             return entity
@@ -60,9 +69,12 @@ class LinkResolver:
                 logger.debug(f"Found entity with path (with .md): {found_path_md.file_path}")
                 return found_path_md
 
-        # search if indicated
+        # In strict mode, don't try fuzzy search - return None if no exact match found
+        if strict:
+            return None
+
+        # 5. Fall back to search for fuzzy matching (only if not in strict mode)
         if use_search and "*" not in clean_text:
-            # 5. Fall back to search for fuzzy matching on title (use text search for prefix matching)
             results = await self.search_service.search(
                 query=SearchQuery(text=clean_text, entity_types=[SearchItemType.ENTITY]),
             )
@@ -101,5 +113,8 @@ class LinkResolver:
             text, alias = text.split("|", 1)
             text = text.strip()
             alias = alias.strip()
+        else:
+            # Strip whitespace from text even if no alias
+            text = text.strip()
 
         return text, alias