thoth-dbmanager 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

thoth_dbmanager/lsh/factory.py

@@ -0,0 +1,76 @@
+"""
+Factory for creating LSH indices from database managers.
+"""
+
+import logging
+from typing import TYPE_CHECKING, Dict, List
+
+if TYPE_CHECKING:
+    from ..ThothDbManager import ThothDbManager
+
+from .manager import LshManager
+
+
+class LshFactory:
+    """Factory for creating LSH indices from any database type."""
+    
+    @staticmethod
+    def create_lsh_from_db(
+        db_manager: "ThothDbManager", 
+        signature_size: int = 30,
+        n_gram: int = 3,
+        threshold: float = 0.5,
+        verbose: bool = True,
+        **kwargs
+    ) -> None:
+        """
+        Create LSH index from any database manager type.
+        
+        This function extracts unique values from the database manager
+        and creates an LSH index using the LshManager.
+        
+        Args:
+            db_manager: Any ThothDbManager implementation
+            signature_size: Size of MinHash signature
+            n_gram: N-gram size for MinHash
+            threshold: LSH similarity threshold
+            verbose: Whether to show progress
+            **kwargs: Additional arguments
+        """
+        if not db_manager.db_directory_path:
+            raise ValueError("Database manager must have a valid db_directory_path")
+        
+        # Get unique values from the database
+        logging.info(f"Extracting unique values from {db_manager.db_id}")
+        unique_values = db_manager.get_unique_values()
+        
+        # Get or create LSH manager
+        lsh_manager = db_manager.lsh_manager
+        if lsh_manager is None:
+            raise ValueError("Could not create LSH manager for database")
+        
+        # Create the LSH index
+        lsh_manager.create_lsh(
+            unique_values=unique_values,
+            signature_size=signature_size,
+            n_gram=n_gram,
+            threshold=threshold,
+            verbose=verbose,
+            **kwargs
+        )
+        
+        logging.info(f"LSH creation completed for {db_manager.db_id}")
+
+
+def make_db_lsh(db_manager: "ThothDbManager", **kwargs) -> None:
+    """
+    Create LSH for any database type (maintains backward compatibility).
+    
+    This function provides backward compatibility with the existing
+    make_db_lsh function signature while using the new architecture.
+    
+    Args:
+        db_manager: Database manager instance
+        **kwargs: LSH creation parameters
+    """
+    LshFactory.create_lsh_from_db(db_manager, **kwargs)

thoth_dbmanager/lsh/manager.py

@@ -0,0 +1,170 @@
+"""
+LSH Manager for database-independent LSH operations.
+"""
+
+import logging
+import pickle
+from pathlib import Path
+from typing import Dict, List, Tuple, Optional, Any
+
+from datasketch import MinHash, MinHashLSH
+
+from .core import create_lsh_index, query_lsh_index
+from .storage import LshStorageStrategy, PickleStorage
+
+
+class LshManager:
+    """
+    Manages LSH operations independently of database implementation.
+    
+    This class handles creation, storage, loading, and querying of LSH indices
+    using a pluggable storage strategy.
+    """
+    
+    def __init__(self, storage_path: Path, storage_strategy: Optional[LshStorageStrategy] = None):
+        """
+        Initialize the LSH manager.
+        
+        Args:
+            storage_path: Base path for LSH storage (directory containing preprocessed folder)
+            storage_strategy: Storage strategy to use (defaults to PickleStorage)
+        """
+        self.storage_path = Path(storage_path)
+        self.storage_strategy = storage_strategy or PickleStorage()
+        self.lsh: Optional[MinHashLSH] = None
+        self.minhashes: Optional[Dict[str, Tuple[MinHash, str, str, str]]] = None
+        
+        # Determine the database ID from the path
+        self.db_id = self.storage_path.name
+        
+        # Set up the preprocessed directory path
+        self.preprocessed_path = self.storage_path / "preprocessed"
+        self.lsh_base_path = self.preprocessed_path / f"{self.db_id}_lsh"
+    
+    def create_lsh(
+        self, 
+        unique_values: Dict[str, Dict[str, List[str]]], 
+        signature_size: int = 30,
+        n_gram: int = 3,
+        threshold: float = 0.5,
+        verbose: bool = True,
+        **kwargs
+    ) -> None:
+        """
+        Create and persist LSH index from unique values.
+        
+        Args:
+            unique_values: Dictionary of unique values from database
+            signature_size: Size of MinHash signature
+            n_gram: N-gram size for MinHash
+            threshold: LSH similarity threshold
+            verbose: Whether to show progress
+            **kwargs: Additional arguments
+        """
+        logging.info(f"Creating LSH for database: {self.db_id}")
+        
+        # Ensure preprocessed directory exists
+        self.preprocessed_path.mkdir(parents=True, exist_ok=True)
+        
+        # Save unique values for reference
+        unique_values_path = self.preprocessed_path / f"{self.db_id}_unique_values.pkl"
+        with open(unique_values_path, "wb") as file:
+            pickle.dump(unique_values, file)
+        logging.info("Saved unique values")
+        
+        # Create LSH index
+        lsh, minhashes = create_lsh_index(
+            unique_values=unique_values,
+            signature_size=signature_size,
+            n_gram=n_gram,
+            threshold=threshold,
+            verbose=verbose
+        )
+        
+        # Store LSH data using the storage strategy
+        self.storage_strategy.save(lsh, minhashes, self.lsh_base_path)
+        logging.info(f"LSH saved to {self.lsh_base_path}")
+        
+        # Keep in memory for immediate use
+        self.lsh = lsh
+        self.minhashes = minhashes
+    
+    def load_lsh(self) -> bool:
+        """
+        Load LSH index from storage.
+        
+        Returns:
+            True if successfully loaded, False otherwise
+        """
+        try:
+            if not self.storage_strategy.exists(self.lsh_base_path):
+                logging.warning(f"LSH files not found at {self.lsh_base_path}")
+                return False
+            
+            lsh_data, minhashes_data = self.storage_strategy.load(self.lsh_base_path)
+            
+            if lsh_data is None or minhashes_data is None:
+                logging.error(f"Failed to load LSH data from {self.lsh_base_path}")
+                return False
+            
+            self.lsh = lsh_data
+            self.minhashes = minhashes_data
+            logging.info(f"LSH loaded successfully for {self.db_id}")
+            return True
+            
+        except Exception as e:
+            logging.error(f"Error loading LSH for {self.db_id}: {e}")
+            return False
+    
+    def query(
+        self, 
+        keyword: str, 
+        signature_size: int = 30,
+        n_gram: int = 3,
+        top_n: int = 10,
+        **kwargs
+    ) -> Dict[str, Dict[str, List[str]]]:
+        """
+        Query the LSH index for similar values.
+        
+        Args:
+            keyword: Search keyword
+            signature_size: Size of MinHash signature
+            n_gram: N-gram size for MinHash
+            top_n: Number of top results to return
+            **kwargs: Additional arguments
+            
+        Returns:
+            Dictionary of similar values organized by table and column
+            
+        Raises:
+            Exception: If LSH is not loaded or query fails
+        """
+        if self.lsh is None or self.minhashes is None:
+            # Try to load LSH if not already loaded
+            if not self.load_lsh():
+                raise Exception(f"Error loading LSH for {self.db_id}")
+        
+        return query_lsh_index(
+            lsh=self.lsh,
+            minhashes=self.minhashes,
+            keyword=keyword,
+            signature_size=signature_size,
+            n_gram=n_gram,
+            top_n=top_n
+        )
+    
+    def is_available(self) -> bool:
+        """
+        Check if LSH data is available (either loaded or stored).
+        
+        Returns:
+            True if LSH is available, False otherwise
+        """
+        return (self.lsh is not None and self.minhashes is not None) or \
+               self.storage_strategy.exists(self.lsh_base_path)
+    
+    def clear(self) -> None:
+        """Clear loaded LSH data from memory."""
+        self.lsh = None
+        self.minhashes = None

thoth_dbmanager/lsh/storage.py

@@ -0,0 +1,96 @@
+"""
+Storage strategies for LSH data persistence.
+"""
+
+import pickle
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Tuple, Optional
+
+
+class LshStorageStrategy(ABC):
+    """Abstract base class for LSH storage strategies."""
+    
+    @abstractmethod
+    def save(self, lsh_data: Any, minhashes_data: Any, base_path: Path) -> None:
+        """
+        Save LSH data and minhashes to storage.
+        
+        Args:
+            lsh_data: The LSH index data
+            minhashes_data: The minhashes data
+            base_path: Base path for storage (without file extension)
+        """
+        pass
+        
+    @abstractmethod
+    def load(self, base_path: Path) -> Tuple[Optional[Any], Optional[Any]]:
+        """
+        Load LSH data and minhashes from storage.
+        
+        Args:
+            base_path: Base path for storage (without file extension)
+            
+        Returns:
+            Tuple of (lsh_data, minhashes_data) or (None, None) if not found
+        """
+        pass
+        
+    @abstractmethod
+    def exists(self, base_path: Path) -> bool:
+        """
+        Check if LSH data exists in storage.
+        
+        Args:
+            base_path: Base path for storage (without file extension)
+            
+        Returns:
+            True if data exists, False otherwise
+        """
+        pass
+
+
+class PickleStorage(LshStorageStrategy):
+    """Pickle-based storage strategy (current implementation)."""
+    
+    def save(self, lsh_data: Any, minhashes_data: Any, base_path: Path) -> None:
+        """Save LSH data using pickle format."""
+        lsh_path = base_path.with_suffix('.pkl')
+        minhashes_path = base_path.parent / f"{base_path.stem}_minhashes.pkl"
+        
+        # Ensure directory exists
+        base_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # Save LSH data
+        with open(lsh_path, 'wb') as f:
+            pickle.dump(lsh_data, f)
+            
+        # Save minhashes data
+        with open(minhashes_path, 'wb') as f:
+            pickle.dump(minhashes_data, f)
+    
+    def load(self, base_path: Path) -> Tuple[Optional[Any], Optional[Any]]:
+        """Load LSH data from pickle files."""
+        lsh_path = base_path.with_suffix('.pkl')
+        minhashes_path = base_path.parent / f"{base_path.stem}_minhashes.pkl"
+        
+        try:
+            # Load LSH data
+            with open(lsh_path, 'rb') as f:
+                lsh_data = pickle.load(f)
+                
+            # Load minhashes data
+            with open(minhashes_path, 'rb') as f:
+                minhashes_data = pickle.load(f)
+                
+            return lsh_data, minhashes_data
+            
+        except (FileNotFoundError, pickle.PickleError):
+            return None, None
+    
+    def exists(self, base_path: Path) -> bool:
+        """Check if both LSH and minhashes pickle files exist."""
+        lsh_path = base_path.with_suffix('.pkl')
+        minhashes_path = base_path.parent / f"{base_path.stem}_minhashes.pkl"
+        
+        return lsh_path.exists() and minhashes_path.exists()

thoth_dbmanager/plugins/__init__.py

@@ -0,0 +1,23 @@
+"""
+Database plugins for Thoth SQL Database Manager.
+"""
+
+# Import all plugins to ensure they are registered
+from .postgresql import PostgreSQLPlugin
+from .sqlite import SQLitePlugin
+from .supabase import SupabasePlugin
+from .mysql import MySQLPlugin
+from .mariadb import MariaDBPlugin
+from .sqlserver import SQLServerPlugin
+from .oracle import OraclePlugin
+
+# This ensures all plugins are registered when the module is imported
+__all__ = [
+    "PostgreSQLPlugin",
+    "SQLitePlugin",
+    "SupabasePlugin",
+    "MySQLPlugin",
+    "MariaDBPlugin",
+    "SQLServerPlugin",
+    "OraclePlugin",
+]