genesis-flow 1.0.0__py3-none-any.whl

mlflow/plugins/builtin/sklearn_plugin.py

@@ -0,0 +1,158 @@
+"""
+Scikit-learn Plugin for Genesis-Flow
+
+Provides scikit-learn integration as a modular plugin.
+"""
+
+import logging
+from typing import Dict, Callable
+
+from mlflow.plugins.base import FrameworkPlugin, PluginMetadata, PluginType
+
+logger = logging.getLogger(__name__)
+
+class SklearnPlugin(FrameworkPlugin):
+    """
+    Scikit-learn framework plugin for Genesis-Flow.
+    
+    Provides scikit-learn model logging, autologging, and deployment capabilities.
+    """
+    
+    @classmethod
+    def get_default_metadata(cls) -> PluginMetadata:
+        """Get default metadata for scikit-learn plugin."""
+        return PluginMetadata(
+            name="sklearn",
+            version="1.0.0",
+            description="Scikit-learn machine learning framework integration",
+            author="Genesis-Flow Team",
+            plugin_type=PluginType.FRAMEWORK,
+            dependencies=["scikit-learn"],
+            optional_dependencies=["numpy", "scipy", "pandas"],
+            min_genesis_flow_version="1.0.0",
+            homepage="https://scikit-learn.org",
+            documentation="https://mlflow.org/docs/latest/models.html#scikit-learn-sklearn",
+            license="Apache 2.0",
+            tags=["machine-learning", "scikit-learn", "sklearn"],
+        )
+    
+    def get_module_path(self) -> str:
+        """Get the module path for scikit-learn integration."""
+        return "mlflow.sklearn"
+    
+    def get_autolog_functions(self) -> Dict[str, Callable]:
+        """Get scikit-learn autologging functions."""
+        try:
+            from mlflow.sklearn import autolog
+            return {
+                "autolog": autolog,
+            }
+        except ImportError:
+            logger.warning("Scikit-learn autolog not available")
+            return {}
+    
+    def get_save_functions(self) -> Dict[str, Callable]:
+        """Get scikit-learn model saving functions."""
+        try:
+            from mlflow.sklearn import log_model, save_model
+            return {
+                "log_model": log_model,
+                "save_model": save_model,
+            }
+        except ImportError:
+            logger.warning("Scikit-learn save functions not available")
+            return {}
+    
+    def get_load_functions(self) -> Dict[str, Callable]:
+        """Get scikit-learn model loading functions."""
+        try:
+            from mlflow.sklearn import load_model
+            return {
+                "load_model": load_model,
+            }
+        except ImportError:
+            logger.warning("Scikit-learn load functions not available")
+            return {}
+    
+    def enable(self) -> bool:
+        """Enable scikit-learn plugin with enhanced functionality."""
+        try:
+            # Call parent enable
+            if not super().enable():
+                return False
+            
+            # Additional scikit-learn-specific setup
+            self._setup_sklearn_environment()
+            
+            self._logger.info("Scikit-learn plugin enabled with full functionality")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to enable scikit-learn plugin: {e}")
+            return False
+    
+    def _setup_sklearn_environment(self):
+        """Setup scikit-learn-specific environment and configurations."""
+        try:
+            import sklearn
+            
+            # Log scikit-learn version
+            self._logger.info(f"Scikit-learn version: {sklearn.__version__}")
+            
+            # Setup hooks for automatic model artifact logging
+            self.register_hook("model_save", self._on_model_save)
+            self.register_hook("model_fit", self._on_model_fit)
+            
+        except Exception as e:
+            self._logger.warning(f"Scikit-learn environment setup warning: {e}")
+    
+    def _on_model_save(self, model, *args, **kwargs):
+        """Hook called when a model is saved."""
+        try:
+            import sklearn.base
+            
+            if isinstance(model, sklearn.base.BaseEstimator):
+                self._logger.debug("Scikit-learn model save detected")
+                # Additional logging or processing can be added here
+                
+        except Exception as e:
+            self._logger.warning(f"Scikit-learn model save hook error: {e}")
+    
+    def _on_model_fit(self, model, *args, **kwargs):
+        """Hook called when a model is fitted."""
+        try:
+            import sklearn.base
+            
+            if isinstance(model, sklearn.base.BaseEstimator):
+                self._logger.debug("Scikit-learn model fit detected")
+                # Could automatically log model parameters here
+                
+        except Exception as e:
+            self._logger.warning(f"Scikit-learn model fit hook error: {e}")
+    
+    def check_dependencies(self) -> bool:
+        """Enhanced dependency checking for scikit-learn."""
+        # Check basic dependencies first
+        if not super().check_dependencies():
+            return False
+        
+        try:
+            import sklearn
+            
+            # Check minimum scikit-learn version
+            from packaging.version import Version
+            min_sklearn_version = "1.0.0"
+            
+            if Version(sklearn.__version__) < Version(min_sklearn_version):
+                self._logger.error(f"Scikit-learn {min_sklearn_version}+ required, found {sklearn.__version__}")
+                return False
+            
+            self._logger.debug(f"Scikit-learn {sklearn.__version__} dependency satisfied")
+            return True
+            
+        except ImportError:
+            self._logger.error("Scikit-learn not installed")
+            return False
+        except Exception as e:
+            self._logger.error(f"Scikit-learn dependency check failed: {e}")
+            return False

mlflow/plugins/builtin/transformers_plugin.py

@@ -0,0 +1,187 @@
+"""
+Transformers Plugin for Genesis-Flow
+
+Provides Hugging Face Transformers integration as a modular plugin.
+"""
+
+import logging
+from typing import Dict, Callable
+
+from mlflow.plugins.base import FrameworkPlugin, PluginMetadata, PluginType
+
+logger = logging.getLogger(__name__)
+
+class TransformersPlugin(FrameworkPlugin):
+    """
+    Hugging Face Transformers framework plugin for Genesis-Flow.
+    
+    Provides transformers model logging, autologging, and deployment capabilities.
+    """
+    
+    @classmethod
+    def get_default_metadata(cls) -> PluginMetadata:
+        """Get default metadata for transformers plugin."""
+        return PluginMetadata(
+            name="transformers",
+            version="1.0.0",
+            description="Hugging Face Transformers NLP framework integration",
+            author="Genesis-Flow Team",
+            plugin_type=PluginType.FRAMEWORK,
+            dependencies=["transformers"],
+            optional_dependencies=["torch", "tensorflow", "sentencepiece", "tokenizers"],
+            min_genesis_flow_version="1.0.0",
+            homepage="https://huggingface.co/transformers",
+            documentation="https://mlflow.org/docs/latest/models.html#transformers-transformers",
+            license="Apache 2.0",
+            tags=["nlp", "transformers", "hugging-face", "llm"],
+        )
+    
+    def get_module_path(self) -> str:
+        """Get the module path for transformers integration."""
+        return "mlflow.transformers"
+    
+    def get_autolog_functions(self) -> Dict[str, Callable]:
+        """Get transformers autologging functions."""
+        try:
+            from mlflow.transformers import autolog
+            return {
+                "autolog": autolog,
+            }
+        except ImportError:
+            logger.warning("Transformers autolog not available")
+            return {}
+    
+    def get_save_functions(self) -> Dict[str, Callable]:
+        """Get transformers model saving functions."""
+        try:
+            from mlflow.transformers import log_model, save_model
+            return {
+                "log_model": log_model,
+                "save_model": save_model,
+            }
+        except ImportError:
+            logger.warning("Transformers save functions not available")
+            return {}
+    
+    def get_load_functions(self) -> Dict[str, Callable]:
+        """Get transformers model loading functions."""
+        try:
+            from mlflow.transformers import load_model
+            return {
+                "load_model": load_model,
+            }
+        except ImportError:
+            logger.warning("Transformers load functions not available")
+            return {}
+    
+    def enable(self) -> bool:
+        """Enable transformers plugin with enhanced functionality."""
+        try:
+            # Call parent enable
+            if not super().enable():
+                return False
+            
+            # Additional transformers-specific setup
+            self._setup_transformers_environment()
+            
+            self._logger.info("Transformers plugin enabled with full functionality")
+            return True
+            
+        except Exception as e:
+            self._logger.error(f"Failed to enable transformers plugin: {e}")
+            return False
+    
+    def _setup_transformers_environment(self):
+        """Setup transformers-specific environment and configurations."""
+        try:
+            import transformers
+            
+            # Log transformers version
+            self._logger.info(f"Transformers version: {transformers.__version__}")
+            
+            # Check for available backends
+            try:
+                import torch
+                self._logger.info(f"PyTorch backend available: {torch.__version__}")
+            except ImportError:
+                pass
+            
+            try:
+                import tensorflow as tf
+                self._logger.info(f"TensorFlow backend available: {tf.__version__}")
+            except ImportError:
+                pass
+            
+            # Setup hooks for automatic model artifact logging
+            self.register_hook("model_save", self._on_model_save)
+            self.register_hook("model_load", self._on_model_load)
+            
+        except Exception as e:
+            self._logger.warning(f"Transformers environment setup warning: {e}")
+    
+    def _on_model_save(self, model, *args, **kwargs):
+        """Hook called when a model is saved."""
+        try:
+            # Check if it's a transformers model
+            if hasattr(model, 'config') and hasattr(model, 'save_pretrained'):
+                self._logger.debug("Transformers model save detected")
+                # Additional logging or processing can be added here
+                
+        except Exception as e:
+            self._logger.warning(f"Transformers model save hook error: {e}")
+    
+    def _on_model_load(self, model_path, *args, **kwargs):
+        """Hook called when a model is loaded."""
+        try:
+            self._logger.debug(f"Transformers model load detected: {model_path}")
+            # Additional processing can be added here
+            
+        except Exception as e:
+            self._logger.warning(f"Transformers model load hook error: {e}")
+    
+    def check_dependencies(self) -> bool:
+        """Enhanced dependency checking for transformers."""
+        # Check basic dependencies first
+        if not super().check_dependencies():
+            return False
+        
+        try:
+            import transformers
+            
+            # Check minimum transformers version
+            from packaging.version import Version
+            min_transformers_version = "4.20.0"
+            
+            if Version(transformers.__version__) < Version(min_transformers_version):
+                self._logger.error(f"Transformers {min_transformers_version}+ required, found {transformers.__version__}")
+                return False
+            
+            # Check that at least one backend is available
+            has_torch = False
+            has_tf = False
+            
+            try:
+                import torch
+                has_torch = True
+            except ImportError:
+                pass
+            
+            try:
+                import tensorflow
+                has_tf = True
+            except ImportError:
+                pass
+            
+            if not (has_torch or has_tf):
+                self._logger.warning("Neither PyTorch nor TensorFlow backend found for Transformers")
+                # Don't fail, but warn
+            
+            self._logger.debug(f"Transformers {transformers.__version__} dependency satisfied")
+            return True
+            
+        except ImportError:
+            self._logger.error("Transformers not installed")
+            return False
+        except Exception as e:
+            self._logger.error(f"Transformers dependency check failed: {e}")
+            return False

mlflow/plugins/cli.py

@@ -0,0 +1,321 @@
+"""
+CLI Commands for Genesis-Flow Plugin Management
+
+Provides command-line interface for managing plugins:
+- List available plugins
+- Enable/disable plugins
+- Get plugin information
+- Plugin statistics
+"""
+
+import logging
+import click
+from typing import Optional
+
+from mlflow.plugins import get_plugin_manager
+from mlflow.plugins.base import PluginType, PluginState
+
+logger = logging.getLogger(__name__)
+
+@click.group(name="plugins")
+def plugins_cli():
+    """Manage Genesis-Flow plugins."""
+    pass
+
+@plugins_cli.command()
+@click.option("--type", "-t", "plugin_type", type=click.Choice([t.value for t in PluginType]), 
+              help="Filter by plugin type")
+@click.option("--state", "-s", type=click.Choice([s.value for s in PluginState]),
+              help="Filter by plugin state")
+@click.option("--detailed", "-d", is_flag=True, help="Show detailed plugin information")
+def list(plugin_type: Optional[str], state: Optional[str], detailed: bool):
+    """List available plugins."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        # Convert string filters to enums
+        type_filter = PluginType(plugin_type) if plugin_type else None
+        state_filter = PluginState(state) if state else None
+        
+        plugins = manager.list_plugins(plugin_type=type_filter, state=state_filter)
+        
+        if not plugins:
+            click.echo("No plugins found matching criteria.")
+            return
+        
+        if detailed:
+            _show_detailed_plugin_list(plugins)
+        else:
+            _show_simple_plugin_list(plugins)
+            
+    except Exception as e:
+        click.echo(f"Error listing plugins: {e}", err=True)
+
+@plugins_cli.command()
+@click.argument("name")
+def info(name: str):
+    """Show detailed information about a plugin."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        plugin = manager.get_plugin(name)
+        if plugin:
+            _show_plugin_info(plugin.get_info())
+        else:
+            # Check if plugin is available but not enabled
+            if manager.is_plugin_available(name):
+                click.echo(f"Plugin '{name}' is available but not enabled.")
+                click.echo(f"Use 'genesis-flow plugins enable {name}' to enable it.")
+            else:
+                click.echo(f"Plugin '{name}' not found.")
+                
+    except Exception as e:
+        click.echo(f"Error getting plugin info: {e}", err=True)
+
+@plugins_cli.command()
+@click.argument("name")
+@click.option("--force", "-f", is_flag=True, help="Force enable even if dependencies are missing")
+def enable(name: str, force: bool):
+    """Enable a plugin."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        if not manager.is_plugin_available(name):
+            click.echo(f"Plugin '{name}' not found.")
+            return
+        
+        if manager.is_plugin_enabled(name):
+            click.echo(f"Plugin '{name}' is already enabled.")
+            return
+        
+        # Check dependencies unless forced
+        if not force:
+            plugin_class = manager._registry.get_plugin_class(name)
+            if plugin_class:
+                # Create temporary instance to check dependencies
+                metadata = manager._get_plugin_metadata(name, plugin_class)
+                temp_instance = plugin_class(metadata)
+                
+                if not temp_instance.check_dependencies():
+                    click.echo(f"Plugin '{name}' has unmet dependencies.")
+                    click.echo("Use --force to enable anyway, or install missing dependencies.")
+                    return
+        
+        if manager.enable_plugin(name):
+            click.echo(f"Plugin '{name}' enabled successfully.")
+        else:
+            click.echo(f"Failed to enable plugin '{name}'.")
+            
+    except Exception as e:
+        click.echo(f"Error enabling plugin: {e}", err=True)
+
+@plugins_cli.command()
+@click.argument("name")
+def disable(name: str):
+    """Disable a plugin."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        if not manager.is_plugin_enabled(name):
+            click.echo(f"Plugin '{name}' is not enabled.")
+            return
+        
+        if manager.disable_plugin(name):
+            click.echo(f"Plugin '{name}' disabled successfully.")
+        else:
+            click.echo(f"Failed to disable plugin '{name}'.")
+            
+    except Exception as e:
+        click.echo(f"Error disabling plugin: {e}", err=True)
+
+@plugins_cli.command()
+@click.argument("name")
+def reload(name: str):
+    """Reload a plugin."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        if not manager.is_plugin_available(name):
+            click.echo(f"Plugin '{name}' not found.")
+            return
+        
+        if manager.reload_plugin(name):
+            click.echo(f"Plugin '{name}' reloaded successfully.")
+        else:
+            click.echo(f"Failed to reload plugin '{name}'.")
+            
+    except Exception as e:
+        click.echo(f"Error reloading plugin: {e}", err=True)
+
+@plugins_cli.command()
+def stats():
+    """Show plugin statistics."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        stats = manager.get_stats()
+        
+        click.echo("Plugin Statistics:")
+        click.echo("=" * 50)
+        click.echo(f"Total Registered: {stats['total_registered']}")
+        click.echo(f"Total Instances:  {stats['total_instances']}")
+        click.echo(f"Enabled:          {stats['enabled']}")
+        click.echo(f"Disabled:         {stats['disabled']}")
+        click.echo(f"Failed:           {stats['failed']}")
+        
+        if stats['by_type']:
+            click.echo("\nBy Type:")
+            for plugin_type, count in stats['by_type'].items():
+                click.echo(f"  {plugin_type:15} {count}")
+                
+    except Exception as e:
+        click.echo(f"Error getting plugin stats: {e}", err=True)
+
+@plugins_cli.command()
+def discover():
+    """Rediscover available plugins."""
+    try:
+        from mlflow.plugins.discovery import refresh_discovery
+        
+        plugins = refresh_discovery()
+        click.echo(f"Discovered {len(plugins)} plugins:")
+        
+        for name in sorted(plugins.keys()):
+            click.echo(f"  - {name}")
+            
+    except Exception as e:
+        click.echo(f"Error discovering plugins: {e}", err=True)
+
+@plugins_cli.command()
+@click.option("--type", "-t", "plugin_type", type=click.Choice([t.value for t in PluginType]),
+              help="Enable all plugins of this type")
+def enable_all(plugin_type: Optional[str]):
+    """Enable all available plugins or all plugins of a specific type."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        type_filter = PluginType(plugin_type) if plugin_type else None
+        plugins = manager.list_plugins(plugin_type=type_filter, state=PluginState.DISCOVERED)
+        
+        if not plugins:
+            click.echo("No plugins available to enable.")
+            return
+        
+        enabled_count = 0
+        for plugin_info in plugins:
+            name = plugin_info['name']
+            if manager.enable_plugin(name):
+                click.echo(f"✓ Enabled {name}")
+                enabled_count += 1
+            else:
+                click.echo(f"✗ Failed to enable {name}")
+        
+        click.echo(f"\nEnabled {enabled_count}/{len(plugins)} plugins.")
+        
+    except Exception as e:
+        click.echo(f"Error enabling plugins: {e}", err=True)
+
+@plugins_cli.command()
+@click.option("--type", "-t", "plugin_type", type=click.Choice([t.value for t in PluginType]),
+              help="Disable all plugins of this type")
+def disable_all(plugin_type: Optional[str]):
+    """Disable all enabled plugins or all plugins of a specific type."""
+    try:
+        manager = get_plugin_manager()
+        manager.initialize()
+        
+        type_filter = PluginType(plugin_type) if plugin_type else None
+        plugins = manager.list_plugins(plugin_type=type_filter, state=PluginState.ENABLED)
+        
+        if not plugins:
+            click.echo("No plugins enabled to disable.")
+            return
+        
+        disabled_count = 0
+        for plugin_info in plugins:
+            name = plugin_info['name']
+            if manager.disable_plugin(name):
+                click.echo(f"✓ Disabled {name}")
+                disabled_count += 1
+            else:
+                click.echo(f"✗ Failed to disable {name}")
+        
+        click.echo(f"\nDisabled {disabled_count}/{len(plugins)} plugins.")
+        
+    except Exception as e:
+        click.echo(f"Error disabling plugins: {e}", err=True)
+
+def _show_simple_plugin_list(plugins):
+    """Show simple plugin list."""
+    click.echo(f"Found {len(plugins)} plugins:")
+    click.echo()
+    
+    # Group by state
+    by_state = {}
+    for plugin in plugins:
+        state = plugin['state']
+        if state not in by_state:
+            by_state[state] = []
+        by_state[state].append(plugin)
+    
+    for state in ['enabled', 'disabled', 'loaded', 'discovered', 'failed']:
+        if state in by_state:
+            click.echo(f"{state.upper()}:")
+            for plugin in by_state[state]:
+                type_info = f" ({plugin['type']})" if 'type' in plugin else ""
+                click.echo(f"  ✓ {plugin['name']}{type_info}")
+            click.echo()
+
+def _show_detailed_plugin_list(plugins):
+    """Show detailed plugin list."""
+    click.echo(f"Found {len(plugins)} plugins:")
+    click.echo("=" * 80)
+    
+    for plugin in plugins:
+        click.echo(f"Name:        {plugin['name']}")
+        click.echo(f"Version:     {plugin.get('version', 'Unknown')}")
+        click.echo(f"Type:        {plugin.get('type', 'Unknown')}")
+        click.echo(f"State:       {plugin['state']}")
+        click.echo(f"Description: {plugin.get('description', 'No description')}")
+        
+        if plugin.get('dependencies'):
+            click.echo(f"Dependencies: {', '.join(plugin['dependencies'])}")
+        
+        if plugin.get('tags'):
+            click.echo(f"Tags:        {', '.join(plugin['tags'])}")
+        
+        click.echo("-" * 80)
+
+def _show_plugin_info(plugin_info):
+    """Show detailed information about a single plugin."""
+    click.echo("Plugin Information:")
+    click.echo("=" * 50)
+    
+    for key, value in plugin_info.items():
+        if isinstance(value, list):
+            if value:
+                click.echo(f"{key.capitalize()}: {', '.join(value)}")
+            else:
+                click.echo(f"{key.capitalize()}: None")
+        else:
+            click.echo(f"{key.capitalize()}: {value}")
+
+# Add to mlflow CLI
+def register_plugin_cli():
+    """Register plugin CLI commands with MLflow CLI."""
+    try:
+        import mlflow.cli
+        
+        # Add plugins command group to mlflow CLI
+        if hasattr(mlflow.cli, 'cli'):
+            mlflow.cli.cli.add_command(plugins_cli)
+            
+    except Exception as e:
+        logger.warning(f"Failed to register plugin CLI: {e}")