memorisdk 1.0.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

memori/database/connectors/base_connector.py

@@ -0,0 +1,283 @@
+"""
+Base database connector interface for Memori
+Provides abstraction layer for different database backends
+"""
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class DatabaseType(str, Enum):
+    """Supported database types"""
+
+    SQLITE = "sqlite"
+    MYSQL = "mysql"
+    POSTGRESQL = "postgresql"
+
+
+class SearchStrategy(str, Enum):
+    """Available search strategies"""
+
+    NATIVE = "native"  # Use database-specific full-text search
+    LIKE_FALLBACK = "like"  # Use LIKE queries with indexing
+    HYBRID = "hybrid"  # Native with fallback to LIKE
+    EXTERNAL = "external"  # External search engine (future)
+
+
+class BaseDatabaseConnector(ABC):
+    """Abstract base class for database connectors"""
+
+    def __init__(self, connection_config: Dict[str, Any]):
+        self.connection_config = connection_config
+        self.database_type = self._detect_database_type()
+
+    @abstractmethod
+    def _detect_database_type(self) -> DatabaseType:
+        """Detect the database type from connection config"""
+        pass
+
+    @abstractmethod
+    def get_connection(self):
+        """Get database connection"""
+        pass
+
+    @abstractmethod
+    def execute_query(
+        self, query: str, params: Optional[List[Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """Execute a query and return results"""
+        pass
+
+    @abstractmethod
+    def execute_insert(self, query: str, params: Optional[List[Any]] = None) -> str:
+        """Execute an insert query and return the inserted row ID"""
+        pass
+
+    @abstractmethod
+    def execute_update(self, query: str, params: Optional[List[Any]] = None) -> int:
+        """Execute an update query and return number of affected rows"""
+        pass
+
+    @abstractmethod
+    def execute_delete(self, query: str, params: Optional[List[Any]] = None) -> int:
+        """Execute a delete query and return number of affected rows"""
+        pass
+
+    @abstractmethod
+    def execute_transaction(
+        self, queries: List[Tuple[str, Optional[List[Any]]]]
+    ) -> bool:
+        """Execute multiple queries in a transaction"""
+        pass
+
+    @abstractmethod
+    def test_connection(self) -> bool:
+        """Test if the database connection is working"""
+        pass
+
+    @abstractmethod
+    def initialize_schema(self, schema_sql: Optional[str] = None):
+        """Initialize database schema"""
+        pass
+
+    @abstractmethod
+    def supports_full_text_search(self) -> bool:
+        """Check if the database supports native full-text search"""
+        pass
+
+    @abstractmethod
+    def create_full_text_index(
+        self, table: str, columns: List[str], index_name: str
+    ) -> str:
+        """Create database-specific full-text search index"""
+        pass
+
+    @abstractmethod
+    def get_database_info(self) -> Dict[str, Any]:
+        """Get database information and capabilities"""
+        pass
+
+
+class BaseSearchAdapter(ABC):
+    """Abstract base class for search adapters"""
+
+    def __init__(self, connector: BaseDatabaseConnector):
+        self.connector = connector
+        self.database_type = connector.database_type
+
+    @abstractmethod
+    def execute_fulltext_search(
+        self,
+        query: str,
+        namespace: str = "default",
+        category_filter: Optional[List[str]] = None,
+        limit: int = 10,
+    ) -> List[Dict[str, Any]]:
+        """Execute full-text search using database-specific methods"""
+        pass
+
+    @abstractmethod
+    def create_search_indexes(self) -> List[str]:
+        """Create search indexes for optimal performance"""
+        pass
+
+    @abstractmethod
+    def translate_search_query(self, query: str) -> str:
+        """Translate search query to database-specific syntax"""
+        pass
+
+    def execute_fallback_search(
+        self,
+        query: str,
+        namespace: str = "default",
+        category_filter: Optional[List[str]] = None,
+        limit: int = 10,
+    ) -> List[Dict[str, Any]]:
+        """Execute fallback LIKE-based search with proper parameterization"""
+        try:
+            # Input validation and sanitization
+            sanitized_query = str(query).strip() if query else ""
+            sanitized_namespace = str(namespace).strip()
+            sanitized_limit = max(1, min(int(limit), 1000))  # Limit between 1 and 1000
+
+            results = []
+
+            # Validate and sanitize category filter
+            sanitized_categories = []
+            if category_filter and isinstance(category_filter, list):
+                sanitized_categories = [
+                    str(cat).strip() for cat in category_filter if cat
+                ]
+
+            # Search short-term memory with parameterized query
+            if sanitized_categories:
+                category_placeholders = ",".join(["?"] * len(sanitized_categories))
+                short_term_query = f"""
+                    SELECT *, 'short_term' as memory_type, 'like_fallback' as search_strategy
+                    FROM short_term_memory
+                    WHERE namespace = ? AND (searchable_content LIKE ? OR summary LIKE ?)
+                    AND category_primary IN ({category_placeholders})
+                    ORDER BY importance_score DESC, created_at DESC
+                    LIMIT ?
+                """
+                short_term_params = (
+                    [
+                        sanitized_namespace,
+                        f"%{sanitized_query}%",
+                        f"%{sanitized_query}%",
+                    ]
+                    + sanitized_categories
+                    + [sanitized_limit]
+                )
+            else:
+                short_term_query = """
+                    SELECT *, 'short_term' as memory_type, 'like_fallback' as search_strategy
+                    FROM short_term_memory
+                    WHERE namespace = ? AND (searchable_content LIKE ? OR summary LIKE ?)
+                    ORDER BY importance_score DESC, created_at DESC
+                    LIMIT ?
+                """
+                short_term_params = [
+                    sanitized_namespace,
+                    f"%{sanitized_query}%",
+                    f"%{sanitized_query}%",
+                    sanitized_limit,
+                ]
+
+            try:
+                results.extend(
+                    self.connector.execute_query(short_term_query, short_term_params)
+                )
+            except Exception:
+                # Log error but continue with long-term search
+                pass
+
+            # Search long-term memory with parameterized query
+            if sanitized_categories:
+                category_placeholders = ",".join(["?"] * len(sanitized_categories))
+                long_term_query = f"""
+                    SELECT *, 'long_term' as memory_type, 'like_fallback' as search_strategy
+                    FROM long_term_memory
+                    WHERE namespace = ? AND (searchable_content LIKE ? OR summary LIKE ?)
+                    AND category_primary IN ({category_placeholders})
+                    ORDER BY importance_score DESC, created_at DESC
+                    LIMIT ?
+                """
+                long_term_params = (
+                    [
+                        sanitized_namespace,
+                        f"%{sanitized_query}%",
+                        f"%{sanitized_query}%",
+                    ]
+                    + sanitized_categories
+                    + [sanitized_limit]
+                )
+            else:
+                long_term_query = """
+                    SELECT *, 'long_term' as memory_type, 'like_fallback' as search_strategy
+                    FROM long_term_memory
+                    WHERE namespace = ? AND (searchable_content LIKE ? OR summary LIKE ?)
+                    ORDER BY importance_score DESC, created_at DESC
+                    LIMIT ?
+                """
+                long_term_params = [
+                    sanitized_namespace,
+                    f"%{sanitized_query}%",
+                    f"%{sanitized_query}%",
+                    sanitized_limit,
+                ]
+
+            try:
+                results.extend(
+                    self.connector.execute_query(long_term_query, long_term_params)
+                )
+            except Exception:
+                # Log error but continue
+                pass
+
+            return results[:sanitized_limit]  # Ensure final limit
+
+        except Exception:
+            # Return empty results on error instead of raising exception
+            return []
+
+
+class BaseSchemaGenerator(ABC):
+    """Abstract base class for database-specific schema generators"""
+
+    def __init__(self, database_type: DatabaseType):
+        self.database_type = database_type
+
+    @abstractmethod
+    def generate_core_schema(self) -> str:
+        """Generate core tables schema"""
+        pass
+
+    @abstractmethod
+    def generate_indexes(self) -> str:
+        """Generate database-specific indexes"""
+        pass
+
+    @abstractmethod
+    def generate_search_setup(self) -> str:
+        """Generate search-related schema (indexes, triggers, etc.)"""
+        pass
+
+    @abstractmethod
+    def get_data_type_mappings(self) -> Dict[str, str]:
+        """Get database-specific data type mappings"""
+        pass
+
+    def generate_full_schema(self) -> str:
+        """Generate complete schema"""
+        schema_parts = [
+            "-- Generated schema for " + self.database_type.value.upper(),
+            "",
+            self.generate_core_schema(),
+            "",
+            self.generate_indexes(),
+            "",
+            self.generate_search_setup(),
+        ]
+        return "\n".join(schema_parts)

memori/database/connectors/mysql_connector.py

@@ -1,22 +1,24 @@
 """
-MySQL connector for Memori v1.0
+MySQL connector for Memori v2.0
+Implements BaseDatabaseConnector interface with FULLTEXT search support
 """
 
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import urlparse
 
 from loguru import logger
 
 from ...utils.exceptions import DatabaseError
+from .base_connector import BaseDatabaseConnector, DatabaseType
 
 
-class MySQLConnector:
-    """MySQL database connector"""
+class MySQLConnector(BaseDatabaseConnector):
+    """MySQL database connector with FULLTEXT search support"""
 
-    def __init__(self, connection_config: Dict[str, str]):
-        """Initialize MySQL connector"""
-        self.connection_config = connection_config
+    def __init__(self, connection_config: Dict[str, Any]):
         self._mysql = None
         self._setup_mysql()
+        super().__init__(connection_config)
 
     def _setup_mysql(self):
         """Setup MySQL connection library"""
@@ -30,17 +32,67 @@ class MySQLConnector:
                 "Install it with: pip install mysql-connector-python"
             )
 
+    def _detect_database_type(self) -> DatabaseType:
+        """Detect database type from connection config"""
+        return DatabaseType.MYSQL
+
+    def _parse_connection_string(self, connection_string: str) -> Dict[str, Any]:
+        """Parse MySQL connection string into connection config"""
+        if connection_string.startswith("mysql://"):
+            parsed = urlparse(connection_string)
+            return {
+                "host": parsed.hostname or "localhost",
+                "port": parsed.port or 3306,
+                "user": parsed.username,
+                "password": parsed.password,
+                "database": parsed.path.lstrip("/") if parsed.path else None,
+            }
+        elif isinstance(self.connection_config, dict):
+            return self.connection_config
+        else:
+            raise DatabaseError(
+                f"Invalid MySQL connection configuration: {connection_string}"
+            )
+
     def get_connection(self):
-        """Get MySQL connection"""
+        """Get MySQL connection with proper configuration"""
         try:
-            conn = self._mysql.connect(**self.connection_config)
+            # Parse connection string if needed
+            if isinstance(self.connection_config, str):
+                config = self._parse_connection_string(self.connection_config)
+            else:
+                config = self.connection_config.copy()
+
+            # Set MySQL-specific options
+            config.update(
+                {
+                    "charset": "utf8mb4",
+                    "collation": "utf8mb4_unicode_ci",
+                    "autocommit": False,
+                    "use_pure": True,  # Use pure Python implementation
+                }
+            )
+
+            conn = self._mysql.connect(**config)
+
+            # Set session variables for optimal performance
+            cursor = conn.cursor()
+            cursor.execute(
+                "SET SESSION sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO'"
+            )
+            cursor.execute("SET SESSION innodb_lock_wait_timeout = 30")
+            cursor.execute(
+                "SET SESSION ft_min_word_len = 3"
+            )  # FULLTEXT minimum word length
+            cursor.close()
+
             return conn
 
         except Exception as e:
             raise DatabaseError(f"Failed to connect to MySQL database: {e}")
 
     def execute_query(
-        self, query: str, params: Optional[List] = None
+        self, query: str, params: Optional[List[Any]] = None
     ) -> List[Dict[str, Any]]:
         """Execute a query and return results"""
         try:
@@ -60,7 +112,7 @@ class MySQLConnector:
         except Exception as e:
             raise DatabaseError(f"Failed to execute query: {e}")
 
-    def execute_insert(self, query: str, params: Optional[List] = None) -> str:
+    def execute_insert(self, query: str, params: Optional[List[Any]] = None) -> str:
         """Execute an insert query and return the inserted row ID"""
         try:
             with self.get_connection() as conn:
@@ -80,7 +132,7 @@ class MySQLConnector:
         except Exception as e:
             raise DatabaseError(f"Failed to execute insert: {e}")
 
-    def execute_update(self, query: str, params: Optional[List] = None) -> int:
+    def execute_update(self, query: str, params: Optional[List[Any]] = None) -> int:
         """Execute an update query and return number of affected rows"""
         try:
             with self.get_connection() as conn:
@@ -100,7 +152,7 @@ class MySQLConnector:
         except Exception as e:
             raise DatabaseError(f"Failed to execute update: {e}")
 
-    def execute_delete(self, query: str, params: Optional[List] = None) -> int:
+    def execute_delete(self, query: str, params: Optional[List[Any]] = None) -> int:
         """Execute a delete query and return number of affected rows"""
         try:
             with self.get_connection() as conn:
@@ -120,13 +172,17 @@ class MySQLConnector:
         except Exception as e:
             raise DatabaseError(f"Failed to execute delete: {e}")
 
-    def execute_transaction(self, queries: List[tuple]) -> bool:
+    def execute_transaction(
+        self, queries: List[Tuple[str, Optional[List[Any]]]]
+    ) -> bool:
         """Execute multiple queries in a transaction"""
         try:
             with self.get_connection() as conn:
                 cursor = conn.cursor()
 
                 try:
+                    conn.start_transaction()
+
                     for query, params in queries:
                         if params:
                             cursor.execute(query, params)
@@ -143,7 +199,7 @@ class MySQLConnector:
                     raise e
 
         except Exception as e:
-            logger.error(f"Transaction failed: {e}")
+            logger.error(f"MySQL transaction failed: {e}")
             return False
 
     def test_connection(self) -> bool:
@@ -151,9 +207,175 @@ class MySQLConnector:
         try:
             with self.get_connection() as conn:
                 cursor = conn.cursor()
-                cursor.execute("SELECT 1")
+                cursor.execute("SELECT 1 as test")
+                result = cursor.fetchone()
+                cursor.close()
+                return result is not None
+        except Exception as e:
+            logger.error(f"MySQL connection test failed: {e}")
+            return False
+
+    def initialize_schema(self, schema_sql: Optional[str] = None):
+        """Initialize database schema"""
+        try:
+            if not schema_sql:
+                # Use MySQL-specific schema
+                from ..schema_generators.mysql_schema_generator import (
+                    MySQLSchemaGenerator,
+                )
+
+                schema_generator = MySQLSchemaGenerator()
+                schema_sql = schema_generator.generate_full_schema()
+
+            # Execute schema using transaction
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+
+                try:
+                    conn.start_transaction()
+
+                    # Split schema into individual statements
+                    statements = self._split_mysql_statements(schema_sql)
+
+                    for statement in statements:
+                        statement = statement.strip()
+                        if statement and not statement.startswith("--"):
+                            cursor.execute(statement)
+
+                    conn.commit()
+                    logger.info("MySQL database schema initialized successfully")
+
+                except Exception as e:
+                    conn.rollback()
+                    logger.error(f"Failed to initialize MySQL schema: {e}")
+                    raise DatabaseError(f"Schema initialization failed: {e}")
+                finally:
+                    cursor.close()
+
+        except Exception as e:
+            logger.error(f"Failed to initialize MySQL schema: {e}")
+            raise DatabaseError(f"Failed to initialize MySQL schema: {e}")
+
+    def _split_mysql_statements(self, schema_sql: str) -> List[str]:
+        """Split SQL schema into individual statements handling MySQL syntax"""
+        statements = []
+        current_statement = []
+
+        for line in schema_sql.split("\n"):
+            line = line.strip()
+
+            # Skip comments and empty lines
+            if not line or line.startswith("--"):
+                continue
+
+            current_statement.append(line)
+
+            # MySQL uses semicolon to end statements
+            if line.endswith(";"):
+                statement = " ".join(current_statement)
+                if statement.strip():
+                    statements.append(statement)
+                current_statement = []
+
+        # Add any remaining statement
+        if current_statement:
+            statement = " ".join(current_statement)
+            if statement.strip():
+                statements.append(statement)
+
+        return statements
+
+    def supports_full_text_search(self) -> bool:
+        """Check if MySQL supports FULLTEXT search"""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+
+                # Check MySQL version and InnoDB support for FULLTEXT
+                cursor.execute("SELECT VERSION() as version")
+                version_result = cursor.fetchone()
+
+                cursor.execute("SHOW ENGINES")
+                engines = cursor.fetchall()
+
                 cursor.close()
-                return True
+
+                # MySQL 5.6+ with InnoDB supports FULLTEXT
+                version = version_result["version"] if version_result else "0.0.0"
+                version_parts = [int(x.split("-")[0]) for x in version.split(".")[:2]]
+
+                innodb_available = any(
+                    engine["Engine"] == "InnoDB"
+                    and engine["Support"] in ("YES", "DEFAULT")
+                    for engine in engines
+                )
+
+                return (
+                    version_parts[0] > 5
+                    or (version_parts[0] == 5 and version_parts[1] >= 6)
+                ) and innodb_available
+
         except Exception as e:
-            logger.error(f"Connection test failed: {e}")
+            logger.warning(f"Could not determine MySQL FULLTEXT support: {e}")
             return False
+
+    def create_full_text_index(
+        self, table: str, columns: List[str], index_name: str
+    ) -> str:
+        """Create MySQL FULLTEXT index"""
+        columns_str = ", ".join(columns)
+        return f"ALTER TABLE {table} ADD FULLTEXT INDEX {index_name} ({columns_str})"
+
+    def get_database_info(self) -> Dict[str, Any]:
+        """Get MySQL database information and capabilities"""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor(dictionary=True)
+
+                info = {}
+
+                # Basic version info
+                cursor.execute("SELECT VERSION() as version")
+                version_result = cursor.fetchone()
+                info["version"] = (
+                    version_result["version"] if version_result else "unknown"
+                )
+
+                # Storage engines
+                cursor.execute("SHOW ENGINES")
+                info["engines"] = cursor.fetchall()
+
+                # Database name
+                cursor.execute("SELECT DATABASE() as db_name")
+                db_result = cursor.fetchone()
+                info["database"] = db_result["db_name"] if db_result else "unknown"
+
+                # Character set info
+                cursor.execute("SHOW VARIABLES LIKE 'character_set_%'")
+                charset_vars = cursor.fetchall()
+                info["character_sets"] = {
+                    var["Variable_name"]: var["Value"] for var in charset_vars
+                }
+
+                # FULLTEXT configuration
+                cursor.execute("SHOW VARIABLES LIKE 'ft_%'")
+                fulltext_vars = cursor.fetchall()
+                info["fulltext_config"] = {
+                    var["Variable_name"]: var["Value"] for var in fulltext_vars
+                }
+
+                # Connection info
+                info["database_type"] = self.database_type.value
+                info["fulltext_support"] = self.supports_full_text_search()
+
+                cursor.close()
+                return info
+
+        except Exception as e:
+            logger.warning(f"Could not get MySQL database info: {e}")
+            return {
+                "database_type": self.database_type.value,
+                "version": "unknown",
+                "fulltext_support": False,
+                "error": str(e),
+            }