thoth-dbmanager 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

thoth_dbmanager/core/interfaces.py

@@ -0,0 +1,271 @@
+"""
+Abstract interfaces for database plugins and adapters.
+"""
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Union
+from ..documents import (
+    BaseThothDbDocument,
+    TableDocument,
+    ColumnDocument,
+    QueryDocument,
+    SchemaDocument,
+    ForeignKeyDocument,
+    IndexDocument,
+    ThothDbType
+)
+
+
+class DbAdapter(ABC):
+    """
+    Abstract adapter interface for database operations.
+    Similar to ThothHaystackVectorStore adapter pattern.
+    """
+    
+    def __init__(self, connection_params: Dict[str, Any]):
+        """
+        Initialize the database adapter.
+        
+        Args:
+            connection_params: Database connection parameters
+        """
+        self.connection_params = connection_params
+        self.connection = None
+        self._initialized = False
+    
+    @abstractmethod
+    def connect(self) -> None:
+        """Establish database connection"""
+        pass
+    
+    @abstractmethod
+    def disconnect(self) -> None:
+        """Close database connection"""
+        pass
+    
+    @abstractmethod
+    def execute_query(self, query: str, params: Optional[Dict] = None, fetch: Union[str, int] = "all", timeout: int = 60) -> Any:
+        """
+        Execute SQL query through adapter.
+        
+        Args:
+            query: SQL query string
+            params: Query parameters
+            fetch: How to fetch results ('all', 'one', or number)
+            timeout: Query timeout in seconds
+            
+        Returns:
+            Query results
+        """
+        pass
+    
+    @abstractmethod
+    def get_tables_as_documents(self) -> List[TableDocument]:
+        """Return tables as document objects"""
+        pass
+    
+    @abstractmethod
+    def get_columns_as_documents(self, table_name: str) -> List[ColumnDocument]:
+        """Return columns as document objects"""
+        pass
+    
+    @abstractmethod
+    def get_foreign_keys_as_documents(self) -> List[ForeignKeyDocument]:
+        """Return foreign keys as document objects"""
+        pass
+    
+    @abstractmethod
+    def get_schemas_as_documents(self) -> List[SchemaDocument]:
+        """Return schemas as document objects"""
+        pass
+    
+    @abstractmethod
+    def get_indexes_as_documents(self, table_name: Optional[str] = None) -> List[IndexDocument]:
+        """Return indexes as document objects"""
+        pass
+    
+    @abstractmethod
+    def get_unique_values(self) -> Dict[str, Dict[str, List[str]]]:
+        """
+        Get unique values from the database.
+        
+        Returns:
+            Dict[str, Dict[str, List[str]]]: Dictionary where:
+                - outer key is table name
+                - inner key is column name
+                - value is list of unique values
+        """
+        pass
+    
+    @abstractmethod
+    def get_example_data(self, table_name: str, number_of_rows: int = 30) -> Dict[str, List[Any]]:
+        """
+        Get example data (most frequent values) for each column in a table.
+        
+        Args:
+            table_name (str): The name of the table.
+            number_of_rows (int, optional): Maximum number of example values to return per column. Defaults to 30.
+            
+        Returns:
+            Dict[str, List[Any]]: A dictionary mapping column names to lists of example values.
+        """
+        pass
+    
+    def health_check(self) -> bool:
+        """Check if database connection is healthy"""
+        try:
+            self.execute_query("SELECT 1", fetch="one")
+            return True
+        except Exception:
+            return False
+    
+    def get_connection_info(self) -> Dict[str, Any]:
+        """Get connection information"""
+        return {
+            "adapter_type": self.__class__.__name__,
+            "connection_params": {k: v for k, v in self.connection_params.items() if k != "password"},
+            "connected": self.connection is not None,
+            "healthy": self.health_check() if self.connection else False
+        }
+
+
+class DbPlugin(ABC):
+    """
+    Abstract plugin interface for database implementations.
+    Each database type should implement this interface.
+    """
+    
+    # Plugin metadata
+    plugin_name: str = ""
+    plugin_version: str = "1.0.0"
+    supported_db_types: List[str] = []
+    required_dependencies: List[str] = []
+    
+    def __init__(self, db_root_path: str, db_mode: str = "dev", **kwargs):
+        """
+        Initialize the database plugin.
+        
+        Args:
+            db_root_path: Path to the database root directory
+            db_mode: Database mode (dev, prod, etc.)
+            **kwargs: Additional plugin-specific parameters
+        """
+        self.db_root_path = db_root_path
+        self.db_mode = db_mode
+        self.adapter: Optional[DbAdapter] = None
+        self._initialized = False
+    
+    @abstractmethod
+    def create_adapter(self, **kwargs) -> DbAdapter:
+        """Create and return a database adapter instance"""
+        pass
+    
+    @abstractmethod
+    def validate_connection_params(self, **kwargs) -> bool:
+        """Validate connection parameters for this plugin"""
+        pass
+    
+    def initialize(self, **kwargs) -> None:
+        """Initialize the plugin with connection parameters"""
+        if not self.validate_connection_params(**kwargs):
+            raise ValueError(f"Invalid connection parameters for {self.plugin_name}")
+        
+        self.adapter = self.create_adapter(**kwargs)
+        self.adapter.connect()
+        self._initialized = True
+    
+    def get_plugin_info(self) -> Dict[str, Any]:
+        """Get plugin metadata"""
+        return {
+            "name": self.plugin_name,
+            "version": self.plugin_version,
+            "supported_db_types": self.supported_db_types,
+            "required_dependencies": self.required_dependencies,
+            "initialized": self._initialized
+        }
+    
+    # Document-based operations
+    def add_table_document(self, doc: TableDocument) -> str:
+        """Add a table document (for metadata storage)"""
+        return doc.id
+    
+    def add_column_document(self, doc: ColumnDocument) -> str:
+        """Add a column document (for metadata storage)"""
+        return doc.id
+    
+    def add_query_document(self, doc: QueryDocument) -> str:
+        """Add a query document (for query history/templates)"""
+        return doc.id
+    
+    def search_documents(self, query: str, doc_type: ThothDbType, top_k: int = 10) -> List[BaseThothDbDocument]:
+        """Search for documents similar to query"""
+        # Default implementation - can be overridden by plugins
+        return []
+    
+    def get_document(self, doc_id: str) -> Optional[BaseThothDbDocument]:
+        """Get document by ID"""
+        # Default implementation - can be overridden by plugins
+        return None
+    
+    def get_documents_by_type(self, doc_type: ThothDbType) -> List[BaseThothDbDocument]:
+        """Get all documents of a specific type"""
+        # Default implementation - can be overridden by plugins
+        return []
+    
+    # Backward compatibility methods - delegate to adapter
+    def execute_sql(self, sql: str, params: Optional[Dict] = None, fetch: Union[str, int] = "all", timeout: int = 60) -> Any:
+        """Execute SQL query (backward compatibility)"""
+        if not self.adapter:
+            raise RuntimeError("Plugin not initialized")
+        return self.adapter.execute_query(sql, params, fetch, timeout)
+    
+    def get_tables(self) -> List[Dict[str, str]]:
+        """Get tables in old format (backward compatibility)"""
+        if not self.adapter:
+            raise RuntimeError("Plugin not initialized")
+        
+        table_docs = self.adapter.get_tables_as_documents()
+        return [
+            {
+                "name": doc.table_name,
+                "comment": doc.comment
+            }
+            for doc in table_docs
+        ]
+    
+    def get_columns(self, table_name: str) -> List[Dict[str, Any]]:
+        """Get columns in old format (backward compatibility)"""
+        if not self.adapter:
+            raise RuntimeError("Plugin not initialized")
+        
+        column_docs = self.adapter.get_columns_as_documents(table_name)
+        return [
+            {
+                "name": doc.column_name,
+                "data_type": doc.data_type,
+                "comment": doc.comment,
+                "is_pk": doc.is_pk
+            }
+            for doc in column_docs
+        ]
+    
+    def get_foreign_keys(self) -> List[Dict[str, str]]:
+        """Get foreign keys in old format (backward compatibility)"""
+        if not self.adapter:
+            raise RuntimeError("Plugin not initialized")
+        
+        fk_docs = self.adapter.get_foreign_keys_as_documents()
+        return [
+            {
+                "source_table_name": doc.source_table_name,
+                "source_column_name": doc.source_column_name,
+                "target_table_name": doc.target_table_name,
+                "target_column_name": doc.target_column_name
+            }
+            for doc in fk_docs
+        ]
+    
+    def get_unique_values(self) -> Dict[str, Dict[str, List[str]]]:
+        """Get unique values (backward compatibility)"""
+        if not self.adapter:
+            raise RuntimeError("Plugin not initialized")
+        return self.adapter.get_unique_values()

thoth_dbmanager/core/registry.py

@@ -0,0 +1,220 @@
+"""
+Plugin registry system for database plugins.
+"""
+import logging
+from typing import Dict, List, Type, Optional, Any
+from .interfaces import DbPlugin
+
+logger = logging.getLogger(__name__)
+
+
+class DbPluginRegistry:
+    """
+    Registry for database plugins.
+    Manages plugin registration, discovery, and instantiation.
+    """
+    
+    _plugins: Dict[str, Type[DbPlugin]] = {}
+    _instances: Dict[tuple, DbPlugin] = {}  # Singleton instances
+    
+    @classmethod
+    def register(cls, db_type: str, plugin_class: Type[DbPlugin]) -> None:
+        """
+        Register a plugin for a specific database type.
+        
+        Args:
+            db_type: Database type identifier (e.g., 'postgresql', 'sqlite')
+            plugin_class: Plugin class implementing DbPlugin interface
+        """
+        if not issubclass(plugin_class, DbPlugin):
+            raise TypeError(f"Plugin class {plugin_class.__name__} must inherit from DbPlugin")
+        
+        cls._plugins[db_type] = plugin_class
+        logger.info(f"Registered plugin {plugin_class.__name__} for database type '{db_type}'")
+    
+    @classmethod
+    def unregister(cls, db_type: str) -> None:
+        """
+        Unregister a plugin for a specific database type.
+        
+        Args:
+            db_type: Database type identifier
+        """
+        if db_type in cls._plugins:
+            plugin_class = cls._plugins.pop(db_type)
+            logger.info(f"Unregistered plugin {plugin_class.__name__} for database type '{db_type}'")
+            
+            # Remove any cached instances
+            keys_to_remove = [key for key in cls._instances.keys() if key[0] == db_type]
+            for key in keys_to_remove:
+                del cls._instances[key]
+    
+    @classmethod
+    def get_plugin_class(cls, db_type: str) -> Type[DbPlugin]:
+        """
+        Get plugin class for a specific database type.
+        
+        Args:
+            db_type: Database type identifier
+            
+        Returns:
+            Plugin class
+            
+        Raises:
+            ValueError: If no plugin is registered for the database type
+        """
+        if db_type not in cls._plugins:
+            available_types = list(cls._plugins.keys())
+            raise ValueError(f"No plugin registered for database type '{db_type}'. Available types: {available_types}")
+        
+        return cls._plugins[db_type]
+    
+    @classmethod
+    def create_plugin(cls, db_type: str, db_root_path: str, db_mode: str = "dev", use_singleton: bool = True, **kwargs) -> DbPlugin:
+        """
+        Create a plugin instance for a specific database type.
+        
+        Args:
+            db_type: Database type identifier
+            db_root_path: Path to database root directory
+            db_mode: Database mode
+            use_singleton: Whether to use singleton pattern
+            **kwargs: Additional plugin-specific parameters
+            
+        Returns:
+            Plugin instance
+        """
+        plugin_class = cls.get_plugin_class(db_type)
+        
+        if use_singleton:
+            # Create instance key for singleton pattern
+            instance_key = (db_type, db_root_path, db_mode, tuple(sorted(kwargs.items())))
+            
+            if instance_key in cls._instances:
+                return cls._instances[instance_key]
+            
+            # Create new instance
+            plugin_instance = plugin_class(db_root_path=db_root_path, db_mode=db_mode, **kwargs)
+            cls._instances[instance_key] = plugin_instance
+            return plugin_instance
+        else:
+            # Create new instance without caching
+            return plugin_class(db_root_path=db_root_path, db_mode=db_mode, **kwargs)
+    
+    @classmethod
+    def list_plugins(cls) -> List[str]:
+        """
+        List all registered database types.
+        
+        Returns:
+            List of registered database type identifiers
+        """
+        return list(cls._plugins.keys())
+    
+    @classmethod
+    def get_plugin_info(cls, db_type: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get information about registered plugins.
+        
+        Args:
+            db_type: Specific database type, or None for all plugins
+            
+        Returns:
+            Plugin information dictionary
+        """
+        if db_type:
+            if db_type not in cls._plugins:
+                raise ValueError(f"No plugin registered for database type '{db_type}'")
+            
+            plugin_class = cls._plugins[db_type]
+            return {
+                "db_type": db_type,
+                "plugin_name": plugin_class.plugin_name,
+                "plugin_version": plugin_class.plugin_version,
+                "supported_db_types": plugin_class.supported_db_types,
+                "required_dependencies": plugin_class.required_dependencies,
+                "class_name": plugin_class.__name__,
+                "module": plugin_class.__module__
+            }
+        else:
+            # Return info for all plugins
+            return {
+                db_type: {
+                    "plugin_name": plugin_class.plugin_name,
+                    "plugin_version": plugin_class.plugin_version,
+                    "supported_db_types": plugin_class.supported_db_types,
+                    "required_dependencies": plugin_class.required_dependencies,
+                    "class_name": plugin_class.__name__,
+                    "module": plugin_class.__module__
+                }
+                for db_type, plugin_class in cls._plugins.items()
+            }
+    
+    @classmethod
+    def validate_plugin(cls, plugin_class: Type[DbPlugin]) -> bool:
+        """
+        Validate that a plugin class implements the required interface.
+        
+        Args:
+            plugin_class: Plugin class to validate
+            
+        Returns:
+            True if valid, False otherwise
+        """
+        try:
+            # Check if it's a subclass of DbPlugin
+            if not issubclass(plugin_class, DbPlugin):
+                logger.error(f"Plugin {plugin_class.__name__} does not inherit from DbPlugin")
+                return False
+            
+            # Check required class attributes
+            required_attrs = ['plugin_name', 'plugin_version', 'supported_db_types']
+            for attr in required_attrs:
+                if not hasattr(plugin_class, attr):
+                    logger.error(f"Plugin {plugin_class.__name__} missing required attribute: {attr}")
+                    return False
+            
+            # Check required methods
+            required_methods = ['create_adapter', 'validate_connection_params']
+            for method in required_methods:
+                if not hasattr(plugin_class, method) or not callable(getattr(plugin_class, method)):
+                    logger.error(f"Plugin {plugin_class.__name__} missing required method: {method}")
+                    return False
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error validating plugin {plugin_class.__name__}: {e}")
+            return False
+    
+    @classmethod
+    def clear_instances(cls) -> None:
+        """Clear all cached plugin instances."""
+        cls._instances.clear()
+        logger.info("Cleared all cached plugin instances")
+    
+    @classmethod
+    def clear_registry(cls) -> None:
+        """Clear the entire plugin registry."""
+        cls._plugins.clear()
+        cls._instances.clear()
+        logger.info("Cleared plugin registry")
+
+
+# Auto-discovery decorator
+def register_plugin(db_type: str):
+    """
+    Decorator to automatically register a plugin.
+    
+    Args:
+        db_type: Database type identifier
+        
+    Example:
+        @register_plugin("postgresql")
+        class PostgreSQLPlugin(DbPlugin):
+            pass
+    """
+    def decorator(plugin_class: Type[DbPlugin]):
+        DbPluginRegistry.register(db_type, plugin_class)
+        return plugin_class
+    return decorator

thoth_dbmanager/documents.py

@@ -0,0 +1,155 @@
+"""
+Document models for Thoth SQL Database Manager.
+Provides type-safe document structures similar to thoth_vdb architecture.
+"""
+from enum import Enum
+from pydantic import BaseModel, Field
+from typing import Any, Dict, List, Optional, Union
+from uuid import uuid4
+
+
+class ThothDbType(Enum):
+    """Types of documents supported by Thoth SQL Database Manager"""
+    TABLE = "table"
+    COLUMN = "column"
+    QUERY = "query"
+    SCHEMA = "schema"
+    FOREIGN_KEY = "foreign_key"
+    INDEX = "index"
+
+
+class BaseThothDbDocument(BaseModel):
+    """Base class for all Thoth database documents"""
+    id: str = Field(default_factory=lambda: str(uuid4()))
+    thoth_type: ThothDbType
+    text: str = ""
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class TableDocument(BaseThothDbDocument):
+    """Document representing a database table"""
+    table_name: str
+    comment: str = ""
+    schema_name: str = "public"
+    row_count: Optional[int] = None
+    thoth_type: ThothDbType = ThothDbType.TABLE
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.text:
+            self.text = f"Table: {self.table_name} in schema {self.schema_name}. {self.comment}"
+
+
+class ColumnDocument(BaseThothDbDocument):
+    """Document representing a database column"""
+    table_name: str
+    column_name: str
+    data_type: str
+    comment: str = ""
+    is_pk: bool = False
+    is_nullable: bool = True
+    default_value: Optional[str] = None
+    max_length: Optional[int] = None
+    schema_name: str = "public"
+    thoth_type: ThothDbType = ThothDbType.COLUMN
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.text:
+            pk_text = " (Primary Key)" if self.is_pk else ""
+            nullable_text = " NOT NULL" if not self.is_nullable else ""
+            self.text = f"Column: {self.table_name}.{self.column_name} ({self.data_type}{nullable_text}){pk_text}. {self.comment}"
+
+
+class QueryDocument(BaseThothDbDocument):
+    """Document representing a SQL query with metadata"""
+    query: str
+    query_type: str = "SELECT"  # SELECT, INSERT, UPDATE, DELETE, etc.
+    description: str = ""
+    parameters: List[str] = Field(default_factory=list)
+    result_columns: List[str] = Field(default_factory=list)
+    execution_time_ms: Optional[float] = None
+    thoth_type: ThothDbType = ThothDbType.QUERY
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.text:
+            self.text = f"{self.query_type} query: {self.description}. SQL: {self.query[:100]}..."
+
+
+class SchemaDocument(BaseThothDbDocument):
+    """Document representing a database schema"""
+    schema_name: str
+    description: str = ""
+    table_count: Optional[int] = None
+    owner: Optional[str] = None
+    thoth_type: ThothDbType = ThothDbType.SCHEMA
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.text:
+            self.text = f"Schema: {self.schema_name}. {self.description}"
+
+
+class ForeignKeyDocument(BaseThothDbDocument):
+    """Document representing a foreign key relationship"""
+    source_table_name: str
+    source_column_name: str
+    target_table_name: str
+    target_column_name: str
+    constraint_name: str = ""
+    schema_name: str = "public"
+    thoth_type: ThothDbType = ThothDbType.FOREIGN_KEY
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.text:
+            self.text = f"Foreign Key: {self.source_table_name}.{self.source_column_name} -> {self.target_table_name}.{self.target_column_name}"
+
+
+class IndexDocument(BaseThothDbDocument):
+    """Document representing a database index"""
+    index_name: str
+    table_name: str
+    columns: List[str]
+    is_unique: bool = False
+    is_primary: bool = False
+    index_type: str = "btree"
+    schema_name: str = "public"
+    thoth_type: ThothDbType = ThothDbType.INDEX
+    
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.text:
+            unique_text = "Unique " if self.is_unique else ""
+            primary_text = "Primary " if self.is_primary else ""
+            self.text = f"{unique_text}{primary_text}Index: {self.index_name} on {self.table_name}({', '.join(self.columns)})"
+
+
+# Type aliases for convenience
+ThothDocument = Union[
+    TableDocument,
+    ColumnDocument,
+    QueryDocument,
+    SchemaDocument,
+    ForeignKeyDocument,
+    IndexDocument
+]
+
+# Document type mapping for factory methods
+DOCUMENT_TYPE_MAP = {
+    ThothDbType.TABLE: TableDocument,
+    ThothDbType.COLUMN: ColumnDocument,
+    ThothDbType.QUERY: QueryDocument,
+    ThothDbType.SCHEMA: SchemaDocument,
+    ThothDbType.FOREIGN_KEY: ForeignKeyDocument,
+    ThothDbType.INDEX: IndexDocument,
+}
+
+
+def create_document(doc_type: ThothDbType, **kwargs) -> BaseThothDbDocument:
+    """Factory function to create documents by type"""
+    document_class = DOCUMENT_TYPE_MAP.get(doc_type)
+    if not document_class:
+        raise ValueError(f"Unsupported document type: {doc_type}")
+    return document_class(**kwargs)