memorisdk 2.0.0__py3-none-any.whl → 2.1.0__py3-none-any.whl

memori/database/search/mysql_search_adapter.py

@@ -3,7 +3,7 @@ MySQL FULLTEXT search adapter for Memori
 Implements MySQL-specific search functionality
 """
 
-from typing import Any, Dict, List, Optional
+from typing import Any
 
 from loguru import logger
 
@@ -49,9 +49,9 @@ class MySQLSearchAdapter(BaseSearchAdapter):
         self,
         query: str,
         namespace: str = "default",
-        category_filter: Optional[List[str]] = None,
+        category_filter: list[str] | None = None,
         limit: int = 10,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Execute MySQL FULLTEXT search or fallback to LIKE search"""
         if self.fulltext_available and query and query.strip():
             try:
@@ -70,9 +70,9 @@ class MySQLSearchAdapter(BaseSearchAdapter):
         self,
         query: str,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Execute MySQL FULLTEXT search"""
         results = []
         translated_query = self.translate_search_query(query)
@@ -160,7 +160,7 @@ class MySQLSearchAdapter(BaseSearchAdapter):
         except:
             return 0.0
 
-    def create_search_indexes(self) -> List[str]:
+    def create_search_indexes(self) -> list[str]:
         """Create MySQL FULLTEXT indexes"""
         commands = [
             # Create FULLTEXT index on short_term_memory
@@ -206,9 +206,9 @@ class MySQLSearchAdapter(BaseSearchAdapter):
         self,
         query: str,
         namespace: str = "default",
-        category_filter: Optional[List[str]] = None,
+        category_filter: list[str] | None = None,
         limit: int = 10,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Execute fallback LIKE-based search with MySQL syntax"""
         results = []
 

memori/database/search/sqlite_search_adapter.py

@@ -4,7 +4,7 @@ Maintains existing FTS5 functionality
 """
 
 import sqlite3
-from typing import Any, Dict, List, Optional
+from typing import Any
 
 from loguru import logger
 
@@ -34,9 +34,9 @@ class SQLiteSearchAdapter(BaseSearchAdapter):
         self,
         query: str,
         namespace: str = "default",
-        category_filter: Optional[List[str]] = None,
+        category_filter: list[str] | None = None,
         limit: int = 10,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Execute FTS5 search or fallback to LIKE search"""
         if self.fts5_available:
             try:
@@ -53,9 +53,9 @@ class SQLiteSearchAdapter(BaseSearchAdapter):
         self,
         query: str,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Execute FTS5 search (adapted from original implementation)"""
         # Build FTS query with category filter - handle empty queries
         if query and query.strip():
@@ -119,7 +119,7 @@ class SQLiteSearchAdapter(BaseSearchAdapter):
 
         return self.connector.execute_query(sql_query, params)
 
-    def create_search_indexes(self) -> List[str]:
+    def create_search_indexes(self) -> list[str]:
         """Create FTS5 virtual table and triggers"""
         if not self.fts5_available:
             logger.warning("FTS5 not available, skipping FTS index creation")

memori/database/search_service.py

@@ -4,7 +4,7 @@ Provides cross-database full-text search capabilities
 """
 
 from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Any
 
 from loguru import logger
 from sqlalchemy import and_, desc, or_, text
@@ -24,10 +24,10 @@ class SearchService:
         self,
         query: str,
         namespace: str = "default",
-        category_filter: Optional[List[str]] = None,
+        category_filter: list[str] | None = None,
         limit: int = 10,
-        memory_types: Optional[List[str]] = None,
-    ) -> List[Dict[str, Any]]:
+        memory_types: list[str] | None = None,
+    ) -> list[dict[str, Any]]:
         """
         Search memories across different database backends
 
@@ -41,7 +41,12 @@ class SearchService:
         Returns:
             List of memory dictionaries with search metadata
         """
+        logger.debug(
+            f"SearchService.search_memories called - query: '{query}', namespace: '{namespace}', database: {self.database_type}, limit: {limit}"
+        )
+
         if not query or not query.strip():
+            logger.debug("Empty query provided, returning recent memories")
             return self._get_recent_memories(
                 namespace, category_filter, limit, memory_types
             )
@@ -52,9 +57,14 @@ class SearchService:
         search_short_term = not memory_types or "short_term" in memory_types
         search_long_term = not memory_types or "long_term" in memory_types
 
+        logger.debug(
+            f"Memory types to search - short_term: {search_short_term}, long_term: {search_long_term}, categories: {category_filter}"
+        )
+
         try:
             # Try database-specific full-text search first
             if self.database_type == "sqlite":
+                logger.debug("Using SQLite FTS5 search strategy")
                 results = self._search_sqlite_fts(
                     query,
                     namespace,
@@ -64,6 +74,7 @@ class SearchService:
                     search_long_term,
                 )
             elif self.database_type == "mysql":
+                logger.debug("Using MySQL FULLTEXT search strategy")
                 results = self._search_mysql_fulltext(
                     query,
                     namespace,
@@ -73,6 +84,7 @@ class SearchService:
                     search_long_term,
                 )
             elif self.database_type == "postgresql":
+                logger.debug("Using PostgreSQL FTS search strategy")
                 results = self._search_postgresql_fts(
                     query,
                     namespace,
@@ -82,8 +94,13 @@ class SearchService:
                     search_long_term,
                 )
 
+            logger.debug(f"Primary search strategy returned {len(results)} results")
+
             # If no results or full-text search failed, fall back to LIKE search
             if not results:
+                logger.debug(
+                    "Primary search returned no results, falling back to LIKE search"
+                )
                 results = self._search_like_fallback(
                     query,
                     namespace,
@@ -94,31 +111,69 @@ class SearchService:
                 )
 
         except Exception as e:
-            logger.warning(f"Full-text search failed: {e}, falling back to LIKE search")
-            results = self._search_like_fallback(
-                query,
-                namespace,
-                category_filter,
-                limit,
-                search_short_term,
-                search_long_term,
+            logger.error(
+                f"Full-text search failed for query '{query}' in namespace '{namespace}': {e}"
+            )
+            logger.debug(
+                f"Full-text search error details: {type(e).__name__}: {str(e)}",
+                exc_info=True,
+            )
+            logger.warning(f"Falling back to LIKE search for query '{query}'")
+            try:
+                results = self._search_like_fallback(
+                    query,
+                    namespace,
+                    category_filter,
+                    limit,
+                    search_short_term,
+                    search_long_term,
+                )
+                logger.debug(f"LIKE fallback search returned {len(results)} results")
+            except Exception as fallback_e:
+                logger.error(
+                    f"LIKE fallback search also failed for query '{query}': {fallback_e}"
+                )
+                results = []
+
+        final_results = self._rank_and_limit_results(results, limit)
+        logger.debug(
+            f"SearchService completed - returning {len(final_results)} final results after ranking and limiting"
+        )
+
+        if final_results:
+            logger.debug(
+                f"Top result: memory_id={final_results[0].get('memory_id')}, score={final_results[0].get('composite_score', 0):.3f}, strategy={final_results[0].get('search_strategy')}"
             )
 
-        return self._rank_and_limit_results(results, limit)
+        return final_results
 
     def _search_sqlite_fts(
         self,
         query: str,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
         search_short_term: bool,
         search_long_term: bool,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Search using SQLite FTS5"""
         try:
+            logger.debug(
+                f"SQLite FTS search starting for query: '{query}' in namespace: '{namespace}'"
+            )
+
+            # Use parameters to validate search scope
+            if not search_short_term and not search_long_term:
+                logger.debug("No memory types specified for search, defaulting to both")
+                search_short_term = search_long_term = True
+
+            logger.debug(
+                f"Search scope - short_term: {search_short_term}, long_term: {search_long_term}"
+            )
+
             # Build FTS query
             fts_query = f'"{query.strip()}"'
+            logger.debug(f"FTS query built: {fts_query}")
 
             # Build category filter
             category_clause = ""
@@ -133,41 +188,69 @@ class SearchService:
                 )
                 for i, cat in enumerate(category_filter):
                     params[f"cat_{i}"] = cat
+                logger.debug(f"Category filter applied: {category_filter}")
 
-            # SQLite FTS5 search query
+            # SQLite FTS5 search query with COALESCE to handle NULL values
             sql_query = f"""
                 SELECT
-                    fts.memory_id, fts.memory_type, fts.category_primary,
-                    CASE
-                        WHEN fts.memory_type = 'short_term' THEN st.processed_data
-                        WHEN fts.memory_type = 'long_term' THEN lt.processed_data
-                    END as processed_data,
-                    CASE
-                        WHEN fts.memory_type = 'short_term' THEN st.importance_score
-                        WHEN fts.memory_type = 'long_term' THEN lt.importance_score
-                        ELSE 0.5
-                    END as importance_score,
-                    CASE
-                        WHEN fts.memory_type = 'short_term' THEN st.created_at
-                        WHEN fts.memory_type = 'long_term' THEN lt.created_at
-                    END as created_at,
-                    fts.summary,
-                    rank as search_score,
+                    fts.memory_id,
+                    fts.memory_type,
+                    fts.category_primary,
+                    COALESCE(
+                        CASE
+                            WHEN fts.memory_type = 'short_term' THEN st.processed_data
+                            WHEN fts.memory_type = 'long_term' THEN lt.processed_data
+                        END,
+                        '{{}}'
+                    ) as processed_data,
+                    COALESCE(
+                        CASE
+                            WHEN fts.memory_type = 'short_term' THEN st.importance_score
+                            WHEN fts.memory_type = 'long_term' THEN lt.importance_score
+                            ELSE 0.5
+                        END,
+                        0.5
+                    ) as importance_score,
+                    COALESCE(
+                        CASE
+                            WHEN fts.memory_type = 'short_term' THEN st.created_at
+                            WHEN fts.memory_type = 'long_term' THEN lt.created_at
+                        END,
+                        datetime('now')
+                    ) as created_at,
+                    COALESCE(fts.summary, '') as summary,
+                    COALESCE(rank, 0.0) as search_score,
                     'sqlite_fts5' as search_strategy
                 FROM memory_search_fts fts
                 LEFT JOIN short_term_memory st ON fts.memory_id = st.memory_id AND fts.memory_type = 'short_term'
                 LEFT JOIN long_term_memory lt ON fts.memory_id = lt.memory_id AND fts.memory_type = 'long_term'
                 WHERE memory_search_fts MATCH :fts_query AND fts.namespace = :namespace
                 {category_clause}
-                ORDER BY rank, importance_score DESC
+                ORDER BY search_score, importance_score DESC
                 LIMIT {limit}
             """
 
+            logger.debug(f"Executing SQLite FTS query with params: {params}")
             result = self.session.execute(text(sql_query), params)
-            return [dict(row) for row in result]
+            rows = [dict(row) for row in result]
+            logger.debug(f"SQLite FTS search returned {len(rows)} results")
+
+            # Log details of first result for debugging
+            if rows:
+                logger.debug(
+                    f"Sample result: memory_id={rows[0].get('memory_id')}, type={rows[0].get('memory_type')}, score={rows[0].get('search_score')}"
+                )
+
+            return rows
 
         except Exception as e:
-            logger.debug(f"SQLite FTS5 search failed: {e}")
+            logger.error(
+                f"SQLite FTS5 search failed for query '{query}' in namespace '{namespace}': {e}"
+            )
+            logger.debug(
+                f"SQLite FTS5 error details: {type(e).__name__}: {str(e)}",
+                exc_info=True,
+            )
             # Roll back the transaction to recover from error state
             self.session.rollback()
             return []
@@ -176,15 +259,23 @@ class SearchService:
         self,
         query: str,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
         search_short_term: bool,
         search_long_term: bool,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Search using MySQL FULLTEXT"""
         results = []
 
         try:
+            # Apply limit proportionally between memory types
+            short_limit = (
+                limit // 2 if search_short_term and search_long_term else limit
+            )
+            long_limit = (
+                limit - short_limit if search_short_term and search_long_term else limit
+            )
+
             # Search short-term memory if requested
             if search_short_term:
                 short_query = self.session.query(ShortTermMemory).filter(
@@ -203,7 +294,7 @@ class SearchService:
                         ShortTermMemory.category_primary.in_(category_filter)
                     )
 
-                # Add relevance score
+                # Add relevance score and limit
                 short_results = self.session.execute(
                     short_query.statement.add_columns(
                         text(
@@ -211,7 +302,7 @@ class SearchService:
                         ).params(query=query),
                         text("'short_term' as memory_type"),
                         text("'mysql_fulltext' as search_strategy"),
-                    )
+                    ).limit(short_limit)
                 ).fetchall()
 
                 results.extend([dict(row) for row in short_results])
@@ -234,7 +325,7 @@ class SearchService:
                         LongTermMemory.category_primary.in_(category_filter)
                     )
 
-                # Add relevance score
+                # Add relevance score and limit
                 long_results = self.session.execute(
                     long_query.statement.add_columns(
                         text(
@@ -242,7 +333,7 @@ class SearchService:
                         ).params(query=query),
                         text("'long_term' as memory_type"),
                         text("'mysql_fulltext' as search_strategy"),
-                    )
+                    ).limit(long_limit)
                 ).fetchall()
 
                 results.extend([dict(row) for row in long_results])
@@ -250,7 +341,13 @@ class SearchService:
             return results
 
         except Exception as e:
-            logger.debug(f"MySQL FULLTEXT search failed: {e}")
+            logger.error(
+                f"MySQL FULLTEXT search failed for query '{query}' in namespace '{namespace}': {e}"
+            )
+            logger.debug(
+                f"MySQL FULLTEXT error details: {type(e).__name__}: {str(e)}",
+                exc_info=True,
+            )
             # Roll back the transaction to recover from error state
             self.session.rollback()
             return []
@@ -259,15 +356,23 @@ class SearchService:
         self,
         query: str,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
         search_short_term: bool,
         search_long_term: bool,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Search using PostgreSQL tsvector"""
         results = []
 
         try:
+            # Apply limit proportionally between memory types
+            short_limit = (
+                limit // 2 if search_short_term and search_long_term else limit
+            )
+            long_limit = (
+                limit - short_limit if search_short_term and search_long_term else limit
+            )
+
             # Prepare query for tsquery - handle spaces and special characters
             # Convert simple query to tsquery format (join words with &)
             tsquery_text = " & ".join(query.split())
@@ -290,7 +395,7 @@ class SearchService:
                         ShortTermMemory.category_primary.in_(category_filter)
                     )
 
-                # Add relevance score
+                # Add relevance score and limit
                 short_results = self.session.execute(
                     short_query.statement.add_columns(
                         text(
@@ -298,7 +403,9 @@ class SearchService:
                         ).params(query=tsquery_text),
                         text("'short_term' as memory_type"),
                         text("'postgresql_fts' as search_strategy"),
-                    ).order_by(text("search_score DESC"))
+                    )
+                    .order_by(text("search_score DESC"))
+                    .limit(short_limit)
                 ).fetchall()
 
                 results.extend([dict(row) for row in short_results])
@@ -321,7 +428,7 @@ class SearchService:
                         LongTermMemory.category_primary.in_(category_filter)
                     )
 
-                # Add relevance score
+                # Add relevance score and limit
                 long_results = self.session.execute(
                     long_query.statement.add_columns(
                         text(
@@ -329,7 +436,9 @@ class SearchService:
                         ).params(query=tsquery_text),
                         text("'long_term' as memory_type"),
                         text("'postgresql_fts' as search_strategy"),
-                    ).order_by(text("search_score DESC"))
+                    )
+                    .order_by(text("search_score DESC"))
+                    .limit(long_limit)
                 ).fetchall()
 
                 results.extend([dict(row) for row in long_results])
@@ -337,7 +446,13 @@ class SearchService:
             return results
 
         except Exception as e:
-            logger.debug(f"PostgreSQL FTS search failed: {e}")
+            logger.error(
+                f"PostgreSQL FTS search failed for query '{query}' in namespace '{namespace}': {e}"
+            )
+            logger.debug(
+                f"PostgreSQL FTS error details: {type(e).__name__}: {str(e)}",
+                exc_info=True,
+            )
             # Roll back the transaction to recover from error state
             self.session.rollback()
             return []
@@ -346,24 +461,47 @@ class SearchService:
         self,
         query: str,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
         search_short_term: bool,
         search_long_term: bool,
-    ) -> List[Dict[str, Any]]:
-        """Fallback LIKE-based search"""
+    ) -> list[dict[str, Any]]:
+        """Fallback LIKE-based search with improved flexibility"""
+        logger.debug(
+            f"Starting LIKE fallback search for query: '{query}' in namespace: '{namespace}'"
+        )
         results = []
-        search_pattern = f"%{query}%"
+
+        # Create multiple search patterns for better matching
+        search_patterns = [
+            f"%{query}%",  # Original full query
+        ]
+
+        # Add individual word patterns for better matching
+        words = query.strip().split()
+        if len(words) > 1:
+            for word in words:
+                if len(word) > 2:  # Skip very short words
+                    search_patterns.append(f"%{word}%")
+
+        logger.debug(f"LIKE search patterns: {search_patterns}")
 
         # Search short-term memory
         if search_short_term:
+            # Build OR conditions for all search patterns
+            search_conditions = []
+            for pattern in search_patterns:
+                search_conditions.extend(
+                    [
+                        ShortTermMemory.searchable_content.like(pattern),
+                        ShortTermMemory.summary.like(pattern),
+                    ]
+                )
+
             short_query = self.session.query(ShortTermMemory).filter(
                 and_(
                     ShortTermMemory.namespace == namespace,
-                    or_(
-                        ShortTermMemory.searchable_content.like(search_pattern),
-                        ShortTermMemory.summary.like(search_pattern),
-                    ),
+                    or_(*search_conditions),
                 )
             )
 
@@ -381,6 +519,8 @@ class SearchService:
                 .all()
             )
 
+            logger.debug(f"LIKE fallback found {len(short_results)} short-term results")
+
             for result in short_results:
                 memory_dict = {
                     "memory_id": result.memory_id,
@@ -397,13 +537,20 @@ class SearchService:
 
         # Search long-term memory
         if search_long_term:
+            # Build OR conditions for all search patterns
+            search_conditions = []
+            for pattern in search_patterns:
+                search_conditions.extend(
+                    [
+                        LongTermMemory.searchable_content.like(pattern),
+                        LongTermMemory.summary.like(pattern),
+                    ]
+                )
+
             long_query = self.session.query(LongTermMemory).filter(
                 and_(
                     LongTermMemory.namespace == namespace,
-                    or_(
-                        LongTermMemory.searchable_content.like(search_pattern),
-                        LongTermMemory.summary.like(search_pattern),
-                    ),
+                    or_(*search_conditions),
                 )
             )
 
@@ -421,6 +568,8 @@ class SearchService:
                 .all()
             )
 
+            logger.debug(f"LIKE fallback found {len(long_results)} long-term results")
+
             for result in long_results:
                 memory_dict = {
                     "memory_id": result.memory_id,
@@ -435,15 +584,18 @@ class SearchService:
                 }
                 results.append(memory_dict)
 
+        logger.debug(
+            f"LIKE fallback search completed, returning {len(results)} total results"
+        )
         return results
 
     def _get_recent_memories(
         self,
         namespace: str,
-        category_filter: Optional[List[str]],
+        category_filter: list[str] | None,
         limit: int,
-        memory_types: Optional[List[str]],
-    ) -> List[Dict[str, Any]]:
+        memory_types: list[str] | None,
+    ) -> list[dict[str, Any]]:
         """Get recent memories when no search query is provided"""
         results = []
 
@@ -515,8 +667,8 @@ class SearchService:
         return results
 
     def _rank_and_limit_results(
-        self, results: List[Dict[str, Any]], limit: int
-    ) -> List[Dict[str, Any]]:
+        self, results: list[dict[str, Any]], limit: int
+    ) -> list[dict[str, Any]]:
         """Rank and limit search results"""
         # Calculate composite score
         for result in results: