flowyml 1.1.0__py3-none-any.whl

flowyml/stacks/plugin_config.py

@@ -0,0 +1,78 @@
+"""Plugin Configuration System.
+
+This module handles the configuration of plugins and adaptation rules,
+allowing users to define how external components map to flowyml.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any
+import yaml
+from pathlib import Path
+
+from flowyml.stacks.components import ComponentType
+from flowyml.stacks.bridge import AdaptationRule
+
+
+@dataclass
+class PluginConfig:
+    """Configuration for a single plugin."""
+
+    name: str
+    source: str  # Import path or bridge URI
+    component_type: str = "orchestrator"  # orchestrator, artifact_store, etc.
+    adaptation: dict[str, Any] = field(default_factory=dict)
+
+
+class PluginManager:
+    """Manages plugin configurations and rules."""
+
+    def __init__(self):
+        self.configs: list[PluginConfig] = []
+
+    def load_from_yaml(self, path: str) -> None:
+        """Load plugin configurations from a YAML file."""
+        path_obj = Path(path)
+        if not path_obj.exists():
+            return
+
+        with open(path) as f:
+            data = yaml.safe_load(f)
+
+        if not data or "plugins" not in data:
+            return
+
+        for p_data in data["plugins"]:
+            self.configs.append(
+                PluginConfig(
+                    name=p_data["name"],
+                    source=p_data["source"],
+                    component_type=p_data.get("type", "orchestrator"),
+                    adaptation=p_data.get("adaptation", {}),
+                ),
+            )
+
+    def get_adaptation_rules(self) -> list[AdaptationRule]:
+        """Convert configurations to adaptation rules."""
+        rules = []
+        for config in self.configs:
+            # Determine target type
+            target_type = ComponentType.ORCHESTRATOR
+            if config.component_type == "artifact_store":
+                target_type = ComponentType.ARTIFACT_STORE
+            elif config.component_type == "container_registry":
+                target_type = ComponentType.CONTAINER_REGISTRY
+
+            # Extract mapping
+            method_mapping = config.adaptation.get("method_mapping", {})
+            attribute_mapping = config.adaptation.get("attribute_mapping", {})
+
+            # Create rule
+            rule = AdaptationRule(
+                source_type=config.source,
+                target_type=target_type,
+                method_mapping=method_mapping,
+                attribute_mapping=attribute_mapping,
+            )
+            rules.append(rule)
+
+        return rules

flowyml/stacks/plugins.py

@@ -0,0 +1,401 @@
+"""Plugin System for Custom Stack Components.
+
+This module provides a robust plugin system that allows users to:
+1. Discover and install plugins from various sources (PyPI, Local)
+2. Manage plugin lifecycles (install, update, remove)
+3. Auto-discover components via entry points
+4. Seamlessly integrate components from other ecosystems via Generic Bridges
+"""
+
+import importlib
+import importlib.util
+import importlib.metadata
+import inspect
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from typing import Any, Optional, Protocol, runtime_checkable
+
+from flowyml.stacks.components import (
+    StackComponent,
+    Orchestrator,
+    ArtifactStore,
+    ContainerRegistry,
+    ComponentType,
+)
+from flowyml.stacks.bridge import GenericBridge, AdaptationRule
+
+
+@dataclass
+class PluginInfo:
+    """Metadata about a plugin."""
+
+    name: str
+    version: str
+    description: str = ""
+    author: str = ""
+    is_installed: bool = False
+    source: str = "pypi"  # "pypi", "local", "github"
+    dependencies: list[str] = field(default_factory=list)
+    components: list[str] = field(default_factory=list)
+
+
+@runtime_checkable
+class PluginBridge(Protocol):
+    """Protocol for plugin bridges that adapt external components."""
+
+    def wrap_component(
+        self,
+        component_class: Any,
+        name: str,
+        config: Optional[dict[str, Any]] = None,
+    ) -> type[StackComponent]:
+        """Wrap an external component class into a flowyml component."""
+        ...
+
+    def is_available(self) -> bool:
+        """Check if the external system is available."""
+        ...
+
+
+class ComponentRegistry:
+    """Registry for stack component plugins.
+
+    Supports:
+    - Auto-discovery via entry points
+    - Manual registration
+    - Generic Bridge system for external integrations
+    - Plugin management (install/uninstall)
+    """
+
+    def __init__(self):
+        self._orchestrators: dict[str, type[Orchestrator]] = {}
+        self._artifact_stores: dict[str, type[ArtifactStore]] = {}
+        self._container_registries: dict[str, type[ContainerRegistry]] = {}
+        self._custom_components: dict[str, type[StackComponent]] = {}
+        self._plugins: dict[str, PluginInfo] = {}
+        self._bridges: dict[str, PluginBridge] = {}
+
+        # Register default generic bridge
+        self._register_default_bridges()
+
+        # Auto-discover plugins
+        self._discover_installed_plugins()
+
+    def _register_default_bridges(self) -> None:
+        """Register default bridges."""
+        # Register a generic bridge that can handle ZenML components via rules
+        zenml_rules = [
+            AdaptationRule(
+                source_type="zenml.orchestrators.base.BaseOrchestrator",
+                target_type=ComponentType.ORCHESTRATOR,
+                method_mapping={"run_pipeline": "run"},
+            ),
+            AdaptationRule(
+                source_type="zenml.artifact_stores.base_artifact_store.BaseArtifactStore",
+                target_type=ComponentType.ARTIFACT_STORE,
+            ),
+            AdaptationRule(
+                source_type="zenml.container_registries.base_container_registry.BaseContainerRegistry",
+                target_type=ComponentType.CONTAINER_REGISTRY,
+            ),
+            # Fallback rule for anything named *Orchestrator
+            AdaptationRule(
+                name_pattern=".*Orchestrator",
+                target_type=ComponentType.ORCHESTRATOR,
+            ),
+        ]
+        self.register_bridge("zenml", GenericBridge(rules=zenml_rules))
+
+        # We can add other default bridges here (e.g. Airflow) without hard deps
+
+    def register_bridge(self, prefix: str, bridge: PluginBridge) -> None:
+        """Register a bridge for a specific URI prefix (e.g., 'zenml:', 'airflow:')."""
+        self._bridges[prefix] = bridge
+
+    def _discover_installed_plugins(self) -> None:
+        """Auto-discover installed plugins via entry points."""
+        try:
+            # Discover flowyml plugins
+            eps = importlib.metadata.entry_points()
+            if hasattr(eps, "select"):
+                flowyml_eps = eps.select(group="flowyml.stack_components")
+            else:
+                flowyml_eps = eps.get("flowyml.stack_components", [])
+
+            for ep in flowyml_eps:
+                try:
+                    component_class = ep.load()
+                    self.register(component_class)
+
+                    # Record plugin info
+                    dist = importlib.metadata.distribution(ep.dist.name)
+                    self._plugins[ep.dist.name] = PluginInfo(
+                        name=ep.dist.name,
+                        version=dist.version,
+                        description=dist.metadata.get("Summary", ""),
+                        author=dist.metadata.get("Author", ""),
+                        is_installed=True,
+                        source="pypi",
+                        components=[ep.name],
+                    )
+                except Exception:
+                    pass
+
+            # Discover Bridges
+            if hasattr(eps, "select"):
+                bridge_eps = eps.select(group="flowyml.bridges")
+            else:
+                bridge_eps = eps.get("flowyml.bridges", [])
+
+            for ep in bridge_eps:
+                try:
+                    bridge_class = ep.load()
+                    self.register_bridge(ep.name, bridge_class())
+                except Exception:
+                    pass
+
+        except Exception:
+            # Entry points not available or error in discovery
+            pass
+
+    def register(self, component_class: type[StackComponent], name: str | None = None) -> None:
+        """Register a stack component.
+
+        Args:
+            component_class: Component class to register
+            name: Optional name override. If None, uses class name in snake_case
+        """
+        if name is None:
+            name = self._class_to_snake_case(component_class.__name__)
+
+        # Determine component type and register
+        if issubclass(component_class, Orchestrator):
+            self._orchestrators[name] = component_class
+        elif issubclass(component_class, ArtifactStore):
+            self._artifact_stores[name] = component_class
+        elif issubclass(component_class, ContainerRegistry):
+            self._container_registries[name] = component_class
+        else:
+            self._custom_components[name] = component_class
+
+    def get_orchestrator(self, name: str) -> type[Orchestrator] | None:
+        """Get orchestrator class by name."""
+        return self._orchestrators.get(name)
+
+    def get_artifact_store(self, name: str) -> type[ArtifactStore] | None:
+        """Get artifact store class by name."""
+        return self._artifact_stores.get(name)
+
+    def get_container_registry(self, name: str) -> type[ContainerRegistry] | None:
+        """Get container registry class by name."""
+        return self._container_registries.get(name)
+
+    def get_component(self, name: str) -> type[StackComponent] | None:
+        """Get any component by name."""
+        return (
+            self._orchestrators.get(name)
+            or self._artifact_stores.get(name)
+            or self._container_registries.get(name)
+            or self._custom_components.get(name)
+        )
+
+    def list_orchestrators(self) -> list[str]:
+        """List all registered orchestrators."""
+        return list(self._orchestrators.keys())
+
+    def list_artifact_stores(self) -> list[str]:
+        """List all registered artifact stores."""
+        return list(self._artifact_stores.keys())
+
+    def list_container_registries(self) -> list[str]:
+        """List all registered container registries."""
+        return list(self._container_registries.keys())
+
+    def list_all(self) -> dict[str, list[str]]:
+        """List all registered components."""
+        return {
+            "orchestrators": self.list_orchestrators(),
+            "artifact_stores": self.list_artifact_stores(),
+            "container_registries": self.list_container_registries(),
+            "custom": list(self._custom_components.keys()),
+        }
+
+    def list_plugins(self) -> list[PluginInfo]:
+        """List all discovered plugins."""
+        return list(self._plugins.values())
+
+    def list_bridges(self) -> list[str]:
+        """List all registered bridges."""
+        return list(self._bridges.keys())
+
+    def load_from_path(self, path: str, class_name: str) -> None:
+        """Load a component from a Python file."""
+        spec = importlib.util.spec_from_file_location("custom_module", path)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Could not load module from {path}")
+
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        component_class = getattr(module, class_name)
+        self.register(component_class)
+
+    def load_from_module(self, module_path: str) -> None:
+        """Load all components from a module."""
+        module = importlib.import_module(module_path)
+
+        for _name, obj in inspect.getmembers(module, inspect.isclass):
+            if issubclass(obj, StackComponent) and obj != StackComponent:
+                self.register(obj)
+
+    def load_via_bridge(self, bridge_name: str, component_path: str, name: str | None = None) -> None:
+        """Load a component via a registered bridge."""
+        bridge = self._bridges.get(bridge_name)
+        if not bridge:
+            raise ValueError(f"Bridge '{bridge_name}' not found. Available: {list(self._bridges.keys())}")
+
+        module_path, class_name = component_path.rsplit(".", 1)
+
+        try:
+            module = importlib.import_module(module_path)
+            component_class = getattr(module, class_name)
+
+            wrapper = bridge.wrap_component(component_class, name or class_name)
+            self.register(wrapper, name or class_name)
+        except ImportError as e:
+            raise ImportError(f"Could not import component via {bridge_name}: {e}")
+
+    def load_plugins_from_config(self, config_path: str) -> None:
+        """Load plugins from a configuration file."""
+        from flowyml.stacks.plugin_config import PluginManager
+
+        manager = PluginManager()
+        manager.load_from_yaml(config_path)
+
+        # Get generic bridge (or create one if not exists)
+        bridge = self._bridges.get("zenml")  # Use the default generic bridge
+        if not bridge or not isinstance(bridge, GenericBridge):
+            # Create a fresh generic bridge if needed
+            bridge = GenericBridge()
+            self.register_bridge("generic", bridge)
+
+        # Register components from config
+        for config in manager.configs:
+            # Create specific rule for this component
+            # Note: In a real implementation, we might want to merge rules or handle them more globally
+            # For now, we'll just use the bridge to wrap it
+
+            try:
+                # Import the source class
+                module_path, class_name = config.source.rsplit(".", 1)
+                module = importlib.import_module(module_path)
+                component_class = getattr(module, class_name)
+
+                # Create rule from config
+                # We need to add this rule to the bridge so it knows how to handle it
+                # Or we can pass the rule directly to wrap_component if we modify GenericBridge
+                # But GenericBridge uses internal rules.
+                # Let's add the rule to the bridge
+
+                # Convert config to rule (simplified)
+                target_type = ComponentType.ORCHESTRATOR
+                if config.component_type == "artifact_store":
+                    target_type = ComponentType.ARTIFACT_STORE
+                elif config.component_type == "container_registry":
+                    target_type = ComponentType.CONTAINER_REGISTRY
+
+                rule = AdaptationRule(
+                    source_type=config.source,
+                    target_type=target_type,
+                    method_mapping=config.adaptation.get("method_mapping", {}),
+                    attribute_mapping=config.adaptation.get("attribute_mapping", {}),
+                )
+
+                # Add rule to bridge
+                if isinstance(bridge, GenericBridge):
+                    bridge.rules.append(rule)
+
+                # Wrap and register
+                wrapper = bridge.wrap_component(component_class, config.name)
+                self.register(wrapper, config.name)
+
+            except Exception as e:
+                print(f"Failed to load plugin {config.name}: {e}")
+
+    def install_plugin(self, package_name: str) -> bool:
+        """Install a plugin package via pip."""
+        try:
+            subprocess.check_call([sys.executable, "-m", "pip", "install", package_name])
+            # Refresh discovery
+            importlib.invalidate_caches()
+            self._discover_installed_plugins()
+            return True
+        except subprocess.CalledProcessError:
+            return False
+
+    @staticmethod
+    def _class_to_snake_case(name: str) -> str:
+        """Convert ClassName to class_name."""
+        import re
+
+        s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+        return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
+
+
+# Global registry instance
+_global_component_registry: ComponentRegistry | None = None
+
+
+def get_component_registry() -> ComponentRegistry:
+    """Get the global component registry."""
+    global _global_component_registry
+    if _global_component_registry is None:
+        _global_component_registry = ComponentRegistry()
+    return _global_component_registry
+
+
+def register_component(component_class=None, name: str | None = None):
+    """Decorator to register a custom component."""
+
+    def wrapper(cls):
+        get_component_registry().register(cls, name)
+        return cls
+
+    if component_class is not None:
+        get_component_registry().register(component_class, name)
+        return component_class
+
+    return wrapper
+
+
+def load_component(source: str, name: str | None = None) -> None:
+    """Load a component from various sources.
+
+    Args:
+        source: Can be:
+            - Module path: "my_package.components"
+            - File path: "/path/to/component.py:ClassName"
+            - Bridge URI: "zenml:zenml.integrations.kubernetes.orchestrators.KubernetesOrchestrator"
+            - Bridge URI: "airflow:airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator"
+        name: Optional name to register as
+    """
+    registry = get_component_registry()
+
+    # Check for bridge prefixes (e.g., "zenml:", "airflow:")
+    if ":" in source and not source.startswith("/") and not source.startswith("."):
+        prefix, path = source.split(":", 1)
+
+        # If prefix is a registered bridge, use it
+        if prefix in registry.list_bridges():
+            registry.load_via_bridge(prefix, path, name)
+            return
+
+        # Special case for file paths that might look like "c:/path" on windows or just "file:Class"
+        # But here we assume "file_path:ClassName" for local loading
+        if ".py" in prefix:
+            registry.load_from_path(prefix, path)
+            return
+
+    # Fallback to module load
+    registry.load_from_module(source)

flowyml/stacks/registry.py

@@ -0,0 +1,226 @@
+"""Stack Registry - Manage and switch between different stacks.
+
+This module provides a registry for storing, loading, and switching
+between different infrastructure stacks.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+from flowyml.stacks.base import Stack
+
+
+class StackRegistry:
+    """Registry for managing multiple stacks.
+
+    The registry allows you to:
+    - Register stacks with unique names
+    - Switch between stacks seamlessly
+    - Save and load stack configurations
+    - List available stacks
+
+    Example:
+        ```python
+        from flowyml.stacks import StackRegistry, LocalStack
+        from flowyml.stacks.gcp import GCPStack
+
+        # Initialize registry
+        registry = StackRegistry()
+
+        # Register stacks
+        local_stack = LocalStack(name="local")
+        registry.register_stack(local_stack)
+
+        gcp_stack = GCPStack(name="production", project_id="my-project", bucket_name="my-artifacts")
+        registry.register_stack(gcp_stack)
+
+        # Switch stacks
+        registry.set_active_stack("local")  # For development
+        registry.set_active_stack("production")  # For production
+
+        # Get active stack
+        active = registry.get_active_stack()
+        ```
+    """
+
+    def __init__(self, config_path: str | None = None):
+        """Initialize stack registry.
+
+        Args:
+            config_path: Path to store stack configurations
+        """
+        self.config_path = Path(config_path or ".flowyml/stacks.json")
+        self.stacks: dict[str, Stack] = {}
+        self.active_stack_name: str | None = None
+
+        # Create config directory
+        self.config_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Load existing configurations
+        self.load()
+
+    def register_stack(self, stack: Stack, set_active: bool = False) -> None:
+        """Register a stack in the registry.
+
+        Args:
+            stack: Stack instance to register
+            set_active: Whether to set this as the active stack
+        """
+        self.stacks[stack.name] = stack
+
+        if set_active or self.active_stack_name is None:
+            self.active_stack_name = stack.name
+
+        # Save configuration
+        self.save()
+
+    def unregister_stack(self, name: str) -> None:
+        """Remove a stack from the registry.
+
+        Args:
+            name: Name of the stack to remove
+        """
+        if name in self.stacks:
+            del self.stacks[name]
+
+            # If this was the active stack, clear it
+            if self.active_stack_name == name:
+                self.active_stack_name = list(self.stacks.keys())[0] if self.stacks else None
+
+            self.save()
+
+    def get_stack(self, name: str) -> Stack | None:
+        """Get a specific stack by name.
+
+        Args:
+            name: Stack name
+
+        Returns:
+            Stack instance or None
+        """
+        return self.stacks.get(name)
+
+    def get_active_stack(self) -> Stack | None:
+        """Get the currently active stack.
+
+        Returns:
+            Active stack instance or None
+        """
+        if self.active_stack_name:
+            return self.stacks.get(self.active_stack_name)
+        return None
+
+    def set_active_stack(self, name: str) -> None:
+        """Set the active stack.
+
+        Args:
+            name: Name of the stack to activate
+        """
+        if name not in self.stacks:
+            raise ValueError(f"Stack '{name}' not found in registry")
+
+        self.active_stack_name = name
+        self.save()
+
+    def list_stacks(self) -> list[str]:
+        """List all registered stacks.
+
+        Returns:
+            List of stack names
+        """
+        return list(self.stacks.keys())
+
+    def describe_stack(self, name: str) -> dict[str, Any]:
+        """Get detailed information about a stack.
+
+        Args:
+            name: Stack name
+
+        Returns:
+            Dictionary with stack details
+        """
+        stack = self.get_stack(name)
+        if not stack:
+            raise ValueError(f"Stack '{name}' not found")
+
+        return {
+            "name": stack.name,
+            "is_active": name == self.active_stack_name,
+            "config": stack.config.to_dict(),
+        }
+
+    def save(self) -> None:
+        """Save stack configurations to disk."""
+        config = {
+            "active_stack": self.active_stack_name,
+            "stacks": {name: self._serialize_stack(stack) for name, stack in self.stacks.items()},
+        }
+
+        with open(self.config_path, "w") as f:
+            json.dump(config, f, indent=2)
+
+    def load(self) -> None:
+        """Load stack configurations from disk."""
+        if not self.config_path.exists():
+            return
+
+        try:
+            with open(self.config_path) as f:
+                config = json.load(f)
+
+            self.active_stack_name = config.get("active_stack")
+
+            # Note: In a full implementation, we would deserialize
+            # and recreate stack objects here. For now, stacks must
+            # be registered programmatically.
+
+        except Exception:
+            pass
+
+    def _serialize_stack(self, stack: Stack) -> dict[str, Any]:
+        """Serialize a stack to dictionary."""
+        return stack.config.to_dict()
+
+    def clear(self) -> None:
+        """Clear all registered stacks."""
+        self.stacks.clear()
+        self.active_stack_name = None
+        self.save()
+
+    def __repr__(self) -> str:
+        active = f" (active: {self.active_stack_name})" if self.active_stack_name else ""
+        return f"StackRegistry({len(self.stacks)} stacks{active})"
+
+
+# Global registry instance
+_global_registry: StackRegistry | None = None
+
+
+def get_registry() -> StackRegistry:
+    """Get the global stack registry instance.
+
+    Returns:
+        Global StackRegistry instance
+    """
+    global _global_registry
+    if _global_registry is None:
+        _global_registry = StackRegistry()
+    return _global_registry
+
+
+def get_active_stack() -> Stack | None:
+    """Get the currently active stack from the global registry.
+
+    Returns:
+        Active stack or None
+    """
+    return get_registry().get_active_stack()
+
+
+def set_active_stack(name: str) -> None:
+    """Set the active stack in the global registry.
+
+    Args:
+        name: Name of the stack to activate
+    """
+    get_registry().set_active_stack(name)