genesis-flow 1.0.0__py3-none-any.whl

mlflow/plugins/__init__.py

@@ -0,0 +1,72 @@
+"""
+Genesis-Flow Plugin Architecture
+
+This module provides a comprehensive plugin system for ML frameworks and extensions.
+Plugins can be dynamically discovered, loaded, enabled/disabled, and managed.
+
+The plugin system supports:
+- Framework plugins (PyTorch, TensorFlow, XGBoost, etc.)
+- Custom logging plugins 
+- Model registry plugins
+- Deployment plugins
+- UI extensions
+
+Example usage:
+    from mlflow.plugins import PluginManager
+    
+    # Get plugin manager
+    manager = PluginManager()
+    
+    # List available plugins
+    plugins = manager.list_plugins()
+    
+    # Enable a plugin
+    manager.enable_plugin("pytorch")
+    
+    # Use the plugin
+    import mlflow.pytorch
+"""
+
+from mlflow.plugins.manager import PluginManager
+from mlflow.plugins.registry import PluginRegistry
+from mlflow.plugins.base import BasePlugin, FrameworkPlugin
+from mlflow.plugins.discovery import discover_plugins, scan_entrypoints
+
+__all__ = [
+    "PluginManager",
+    "PluginRegistry", 
+    "BasePlugin",
+    "FrameworkPlugin",
+    "discover_plugins",
+    "scan_entrypoints",
+]
+
+# Global plugin manager instance
+_plugin_manager = None
+
+def get_plugin_manager() -> PluginManager:
+    """Get the global plugin manager instance."""
+    global _plugin_manager
+    if _plugin_manager is None:
+        _plugin_manager = PluginManager()
+    return _plugin_manager
+
+def list_plugins():
+    """List all available plugins."""
+    return get_plugin_manager().list_plugins()
+
+def enable_plugin(name: str):
+    """Enable a plugin by name."""
+    return get_plugin_manager().enable_plugin(name)
+
+def disable_plugin(name: str):
+    """Disable a plugin by name."""
+    return get_plugin_manager().disable_plugin(name)
+
+def is_plugin_enabled(name: str) -> bool:
+    """Check if a plugin is enabled."""
+    return get_plugin_manager().is_plugin_enabled(name)
+
+def get_plugin(name: str):
+    """Get a plugin instance by name."""
+    return get_plugin_manager().get_plugin(name)

mlflow/plugins/base.py

@@ -0,0 +1,358 @@
+"""
+Base Plugin Classes for Genesis-Flow Plugin System
+
+Defines the interface and base classes that all plugins must implement.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, List, Optional, Any, Callable
+from dataclasses import dataclass
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+class PluginType(Enum):
+    """Types of plugins supported by Genesis-Flow."""
+    FRAMEWORK = "framework"      # ML framework integration (PyTorch, TensorFlow, etc.)
+    LOGGING = "logging"         # Custom logging backends
+    MODEL_REGISTRY = "registry" # Model registry integrations
+    DEPLOYMENT = "deployment"   # Deployment platforms
+    ARTIFACT = "artifact"       # Artifact storage backends
+    UI = "ui"                   # UI extensions
+    CUSTOM = "custom"           # Custom extensions
+
+@dataclass
+class PluginMetadata:
+    """Metadata for a plugin."""
+    name: str
+    version: str
+    description: str
+    author: str
+    plugin_type: PluginType
+    dependencies: List[str]
+    optional_dependencies: List[str]
+    min_genesis_flow_version: str
+    max_genesis_flow_version: Optional[str] = None
+    homepage: Optional[str] = None
+    documentation: Optional[str] = None
+    license: Optional[str] = None
+    tags: List[str] = None
+
+    def __post_init__(self):
+        if self.tags is None:
+            self.tags = []
+
+class PluginState(Enum):
+    """States a plugin can be in."""
+    DISCOVERED = "discovered"
+    LOADED = "loaded"
+    ENABLED = "enabled" 
+    DISABLED = "disabled"
+    FAILED = "failed"
+
+class BasePlugin(ABC):
+    """
+    Base class for all Genesis-Flow plugins.
+    
+    All plugins must inherit from this class and implement the required methods.
+    """
+    
+    def __init__(self, metadata: PluginMetadata):
+        self.metadata = metadata
+        self.state = PluginState.DISCOVERED
+        self._logger = logging.getLogger(f"mlflow.plugins.{metadata.name}")
+        self._hooks = {}
+        
+    @abstractmethod
+    def load(self) -> bool:
+        """
+        Load the plugin and perform any necessary initialization.
+        
+        Returns:
+            True if loading was successful, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def enable(self) -> bool:
+        """
+        Enable the plugin for use.
+        
+        Returns:
+            True if enabling was successful, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def disable(self) -> bool:
+        """
+        Disable the plugin.
+        
+        Returns:
+            True if disabling was successful, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def unload(self) -> bool:
+        """
+        Unload the plugin and clean up resources.
+        
+        Returns:
+            True if unloading was successful, False otherwise
+        """
+        pass
+    
+    def is_compatible(self, genesis_flow_version: str) -> bool:
+        """
+        Check if the plugin is compatible with the current Genesis-Flow version.
+        
+        Args:
+            genesis_flow_version: Current Genesis-Flow version
+            
+        Returns:
+            True if compatible, False otherwise
+        """
+        from packaging.version import Version
+        
+        try:
+            current = Version(genesis_flow_version)
+            min_version = Version(self.metadata.min_genesis_flow_version)
+            
+            if current < min_version:
+                return False
+                
+            if self.metadata.max_genesis_flow_version:
+                max_version = Version(self.metadata.max_genesis_flow_version)
+                if current > max_version:
+                    return False
+                    
+            return True
+            
+        except Exception as e:
+            self._logger.warning(f"Version compatibility check failed: {e}")
+            return False
+    
+    def check_dependencies(self) -> bool:
+        """
+        Check if all required dependencies are available.
+        
+        Returns:
+            True if all dependencies are satisfied, False otherwise
+        """
+        try:
+            import importlib
+            
+            for dep in self.metadata.dependencies:
+                try:
+                    importlib.import_module(dep)
+                except ImportError:
+                    self._logger.error(f"Required dependency '{dep}' not found")
+                    return False
+            
+            # Check optional dependencies and log warnings
+            for dep in self.metadata.optional_dependencies:
+                try:
+                    importlib.import_module(dep)
+                except ImportError:
+                    self._logger.warning(f"Optional dependency '{dep}' not found")
+            
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Dependency check failed: {e}")
+            return False
+    
+    def register_hook(self, event: str, callback: Callable):
+        """Register a callback for a specific event."""
+        if event not in self._hooks:
+            self._hooks[event] = []
+        self._hooks[event].append(callback)
+    
+    def trigger_hook(self, event: str, *args, **kwargs):
+        """Trigger all callbacks for a specific event."""
+        if event in self._hooks:
+            for callback in self._hooks[event]:
+                try:
+                    callback(*args, **kwargs)
+                except Exception as e:
+                    self._logger.error(f"Hook callback failed for {event}: {e}")
+    
+    def get_info(self) -> Dict[str, Any]:
+        """Get plugin information."""
+        return {
+            "name": self.metadata.name,
+            "version": self.metadata.version,
+            "description": self.metadata.description,
+            "author": self.metadata.author,
+            "type": self.metadata.plugin_type.value,
+            "state": self.state.value,
+            "dependencies": self.metadata.dependencies,
+            "optional_dependencies": self.metadata.optional_dependencies,
+            "tags": self.metadata.tags,
+        }
+
+class FrameworkPlugin(BasePlugin):
+    """
+    Base class for ML framework plugins.
+    
+    Framework plugins provide integration with ML libraries like PyTorch, TensorFlow, etc.
+    """
+    
+    def __init__(self, metadata: PluginMetadata):
+        super().__init__(metadata)
+        self._module_path = None
+        self._lazy_loader = None
+    
+    @abstractmethod
+    def get_module_path(self) -> str:
+        """
+        Get the module path for the framework integration.
+        
+        Returns:
+            Module path (e.g., "mlflow.pytorch")
+        """
+        pass
+    
+    @abstractmethod
+    def get_autolog_functions(self) -> Dict[str, Callable]:
+        """
+        Get autologging functions provided by this framework.
+        
+        Returns:
+            Dict mapping function names to callables
+        """
+        pass
+    
+    @abstractmethod
+    def get_save_functions(self) -> Dict[str, Callable]:
+        """
+        Get model saving functions provided by this framework.
+        
+        Returns:
+            Dict mapping function names to callables
+        """
+        pass
+    
+    @abstractmethod
+    def get_load_functions(self) -> Dict[str, Callable]:
+        """
+        Get model loading functions provided by this framework.
+        
+        Returns:
+            Dict mapping function names to callables
+        """
+        pass
+    
+    def load(self) -> bool:
+        """Load the framework plugin."""
+        try:
+            if not self.check_dependencies():
+                self.state = PluginState.FAILED
+                return False
+            
+            self._module_path = self.get_module_path()
+            
+            # Create lazy loader for the framework
+            from mlflow.utils.lazy_load import LazyLoader
+            import mlflow
+            
+            self._lazy_loader = LazyLoader(
+                self._module_path,
+                mlflow.__dict__,
+                self._module_path
+            )
+            
+            self.state = PluginState.LOADED
+            self._logger.info(f"Framework plugin {self.metadata.name} loaded")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to load framework plugin {self.metadata.name}: {e}")
+            self.state = PluginState.FAILED
+            return False
+    
+    def enable(self) -> bool:
+        """Enable the framework plugin."""
+        try:
+            if self.state != PluginState.LOADED:
+                if not self.load():
+                    return False
+            
+            # Register the lazy loader in mlflow namespace
+            import mlflow
+            setattr(mlflow, self.metadata.name, self._lazy_loader)
+            
+            self.state = PluginState.ENABLED
+            self._logger.info(f"Framework plugin {self.metadata.name} enabled")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to enable framework plugin {self.metadata.name}: {e}")
+            self.state = PluginState.FAILED
+            return False
+    
+    def disable(self) -> bool:
+        """Disable the framework plugin."""
+        try:
+            import mlflow
+            
+            if hasattr(mlflow, self.metadata.name):
+                delattr(mlflow, self.metadata.name)
+            
+            self.state = PluginState.LOADED
+            self._logger.info(f"Framework plugin {self.metadata.name} disabled")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to disable framework plugin {self.metadata.name}: {e}")
+            return False
+    
+    def unload(self) -> bool:
+        """Unload the framework plugin."""
+        try:
+            self.disable()
+            
+            self._lazy_loader = None
+            self._module_path = None
+            
+            self.state = PluginState.DISCOVERED
+            self._logger.info(f"Framework plugin {self.metadata.name} unloaded")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to unload framework plugin {self.metadata.name}: {e}")
+            return False
+
+class LoggingPlugin(BasePlugin):
+    """Base class for custom logging backend plugins."""
+    
+    @abstractmethod
+    def get_logging_backend(self):
+        """Get the logging backend implementation."""
+        pass
+
+class ModelRegistryPlugin(BasePlugin):
+    """Base class for model registry integration plugins."""
+    
+    @abstractmethod
+    def get_registry_backend(self):
+        """Get the model registry backend implementation."""
+        pass
+
+class DeploymentPlugin(BasePlugin):
+    """Base class for deployment platform plugins."""
+    
+    @abstractmethod
+    def get_deployment_backend(self):
+        """Get the deployment backend implementation."""
+        pass
+
+class ArtifactPlugin(BasePlugin):
+    """Base class for artifact storage plugins."""
+    
+    @abstractmethod
+    def get_artifact_repository(self):
+        """Get the artifact repository implementation."""
+        pass

mlflow/plugins/builtin/__init__.py

@@ -0,0 +1,24 @@
+"""
+Built-in Plugins for Genesis-Flow
+
+This module contains all built-in plugins that ship with Genesis-Flow.
+These plugins provide integration with essential ML frameworks.
+"""
+
+from mlflow.plugins.builtin.pytorch_plugin import PyTorchPlugin
+from mlflow.plugins.builtin.sklearn_plugin import SklearnPlugin
+from mlflow.plugins.builtin.transformers_plugin import TransformersPlugin
+
+# Registry of all built-in plugins
+BUILTIN_PLUGINS = {
+    "pytorch": PyTorchPlugin,
+    "sklearn": SklearnPlugin,
+    "transformers": TransformersPlugin,
+}
+
+__all__ = [
+    "BUILTIN_PLUGINS",
+    "PyTorchPlugin",
+    "SklearnPlugin", 
+    "TransformersPlugin",
+]

mlflow/plugins/builtin/pytorch_plugin.py

@@ -0,0 +1,150 @@
+"""
+PyTorch Plugin for Genesis-Flow
+
+Provides PyTorch integration as a modular plugin that can be enabled/disabled.
+"""
+
+import logging
+from typing import Dict, Callable
+
+from mlflow.plugins.base import FrameworkPlugin, PluginMetadata, PluginType
+
+logger = logging.getLogger(__name__)
+
+class PyTorchPlugin(FrameworkPlugin):
+    """
+    PyTorch framework plugin for Genesis-Flow.
+    
+    Provides PyTorch model logging, autologging, and deployment capabilities.
+    """
+    
+    @classmethod
+    def get_default_metadata(cls) -> PluginMetadata:
+        """Get default metadata for PyTorch plugin."""
+        return PluginMetadata(
+            name="pytorch",
+            version="1.0.0",
+            description="PyTorch deep learning framework integration",
+            author="Genesis-Flow Team",
+            plugin_type=PluginType.FRAMEWORK,
+            dependencies=["torch"],
+            optional_dependencies=["torchvision", "torchaudio"],
+            min_genesis_flow_version="1.0.0",
+            homepage="https://pytorch.org",
+            documentation="https://mlflow.org/docs/latest/models.html#pytorch-pytorch",
+            license="Apache 2.0",
+            tags=["deep-learning", "neural-networks", "pytorch"],
+        )
+    
+    def get_module_path(self) -> str:
+        """Get the module path for PyTorch integration."""
+        return "mlflow.pytorch"
+    
+    def get_autolog_functions(self) -> Dict[str, Callable]:
+        """Get PyTorch autologging functions."""
+        try:
+            from mlflow.pytorch import autolog
+            return {
+                "autolog": autolog,
+            }
+        except ImportError:
+            logger.warning("PyTorch autolog not available")
+            return {}
+    
+    def get_save_functions(self) -> Dict[str, Callable]:
+        """Get PyTorch model saving functions."""
+        try:
+            from mlflow.pytorch import log_model, save_model
+            return {
+                "log_model": log_model,
+                "save_model": save_model,
+            }
+        except ImportError:
+            logger.warning("PyTorch save functions not available")
+            return {}
+    
+    def get_load_functions(self) -> Dict[str, Callable]:
+        """Get PyTorch model loading functions."""
+        try:
+            from mlflow.pytorch import load_model
+            return {
+                "load_model": load_model,
+            }
+        except ImportError:
+            logger.warning("PyTorch load functions not available")
+            return {}
+    
+    def enable(self) -> bool:
+        """Enable PyTorch plugin with enhanced functionality."""
+        try:
+            # Call parent enable
+            if not super().enable():
+                return False
+            
+            # Additional PyTorch-specific setup
+            self._setup_pytorch_environment()
+            
+            self._logger.info("PyTorch plugin enabled with full functionality")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to enable PyTorch plugin: {e}")
+            return False
+    
+    def _setup_pytorch_environment(self):
+        """Setup PyTorch-specific environment and configurations."""
+        try:
+            import torch
+            
+            # Log PyTorch version and CUDA availability
+            self._logger.info(f"PyTorch version: {torch.__version__}")
+            self._logger.info(f"CUDA available: {torch.cuda.is_available()}")
+            
+            if torch.cuda.is_available():
+                self._logger.info(f"CUDA version: {torch.version.cuda}")
+                self._logger.info(f"GPU count: {torch.cuda.device_count()}")
+            
+            # Setup hook for automatic model artifact logging
+            self.register_hook("model_save", self._on_model_save)
+            
+        except Exception as e:
+            self._logger.warning(f"PyTorch environment setup warning: {e}")
+    
+    def _on_model_save(self, model, *args, **kwargs):
+        """Hook called when a model is saved."""
+        try:
+            import torch
+            
+            if isinstance(model, torch.nn.Module):
+                self._logger.debug("PyTorch model save detected")
+                # Additional logging or processing can be added here
+                
+        except Exception as e:
+            self._logger.warning(f"PyTorch model save hook error: {e}")
+    
+    def check_dependencies(self) -> bool:
+        """Enhanced dependency checking for PyTorch."""
+        # Check basic dependencies first
+        if not super().check_dependencies():
+            return False
+        
+        try:
+            import torch
+            
+            # Check minimum PyTorch version
+            from packaging.version import Version
+            min_pytorch_version = "1.9.0"
+            
+            if Version(torch.__version__) < Version(min_pytorch_version):
+                self._logger.error(f"PyTorch {min_pytorch_version}+ required, found {torch.__version__}")
+                return False
+            
+            self._logger.debug(f"PyTorch {torch.__version__} dependency satisfied")
+            return True
+            
+        except ImportError:
+            self._logger.error("PyTorch not installed")
+            return False
+        except Exception as e:
+            self._logger.error(f"PyTorch dependency check failed: {e}")
+            return False