flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/plugins/config.py

@@ -0,0 +1,478 @@
+"""FlowyML Plugin Configuration System.
+
+This module provides YAML-based configuration for plugins, allowing
+users to define their stack configuration in a file and use plugins
+without manual setup in code.
+
+Usage:
+    # flowyml.yaml
+    plugins:
+      experiment_tracker:
+        type: mlflow
+        tracking_uri: http://localhost:5000
+        experiment_name: my_experiments
+
+      artifact_store:
+        type: gcs
+        bucket: my-ml-artifacts
+        prefix: experiments/
+        project: my-gcp-project
+
+      orchestrator:
+        type: vertex_ai
+        project: my-gcp-project
+        location: us-central1
+
+    # In code - just use
+    from flowyml.plugins.config import get_tracker, get_artifact_store
+
+    tracker = get_tracker()  # Uses config from flowyml.yaml
+    tracker.start_run("my_run")
+"""
+
+import os
+import yaml
+import logging
+from pathlib import Path
+from typing import Any
+
+from flowyml.plugins.manager import get_manager
+from flowyml.plugins.base import (
+    BasePlugin,
+    ExperimentTracker,
+    ArtifactStorePlugin,
+    OrchestratorPlugin,
+    ContainerRegistryPlugin,
+    FeatureStorePlugin,
+    DataValidatorPlugin,
+    AlerterPlugin,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Default config file names to search for
+CONFIG_FILE_NAMES = [
+    "flowyml.yaml",
+    "flowyml.yml",
+    ".flowyml.yaml",
+    ".flowyml.yml",
+]
+
+
+class PluginConfig:
+    """Manages plugin configuration from YAML files.
+
+    Configuration can be loaded from:
+    1. A specific file path
+    2. Auto-discovered from current directory
+    3. Environment variable FLOWYML_CONFIG
+
+    Example flowyml.yaml:
+
+        plugins:
+          experiment_tracker:
+            type: mlflow
+            tracking_uri: http://localhost:5000
+            experiment_name: my_experiments
+
+          artifact_store:
+            type: s3
+            bucket: my-ml-artifacts
+            prefix: experiments/
+
+          orchestrator:
+            type: kubernetes
+            namespace: ml-pipelines
+
+          container_registry:
+            type: gcr
+            project: my-gcp-project
+            location: us-central1
+            repository: ml-images
+    """
+
+    def __init__(self, config_path: str = None):
+        """Initialize the plugin configuration.
+
+        Args:
+            config_path: Optional path to config file. If not provided,
+                        auto-discovers from current directory.
+        """
+        self._config_path = config_path
+        self._config: dict = {}
+        self._manager = get_manager()
+        self._instances: dict[str, BasePlugin] = {}
+
+        # Load config
+        self._load_config()
+
+    def _load_config(self) -> None:
+        """Load configuration from file."""
+        config_path = self._find_config_file()
+
+        if config_path:
+            try:
+                with open(config_path) as f:
+                    raw_config = yaml.safe_load(f) or {}
+
+                # Substitute environment variables
+                self._config = self._substitute_env_vars(raw_config)
+                self._config_path = config_path
+                logger.info(f"Loaded plugin config from: {config_path}")
+            except Exception as e:
+                logger.warning(f"Failed to load config from {config_path}: {e}")
+                self._config = {}
+        else:
+            logger.debug("No config file found, using defaults")
+            self._config = {}
+
+    def _substitute_env_vars(self, obj: Any) -> Any:
+        """Recursively substitute environment variables in config.
+
+        Supports ${VAR_NAME} and ${VAR_NAME:-default} syntax.
+
+        Args:
+            obj: Config object (dict, list, or scalar).
+
+        Returns:
+            Object with environment variables substituted.
+        """
+        import re
+
+        if isinstance(obj, dict):
+            return {k: self._substitute_env_vars(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self._substitute_env_vars(item) for item in obj]
+        elif isinstance(obj, str):
+            # Pattern: ${VAR_NAME} or ${VAR_NAME:-default}
+            pattern = r"\$\{([^}:]+)(?::-([^}]*))?\}"
+
+            def replace(match):
+                var_name = match.group(1)
+                default = match.group(2)
+                value = os.environ.get(var_name)
+                if value is not None:
+                    return value
+                elif default is not None:
+                    return default
+                else:
+                    logger.warning(f"Environment variable '{var_name}' not set")
+                    return match.group(0)  # Keep original if not found
+
+            return re.sub(pattern, replace, obj)
+        else:
+            return obj
+
+    def _find_config_file(self) -> str | None:
+        """Find the configuration file.
+
+        Searches in order:
+        1. Explicit path provided to constructor
+        2. FLOWYML_CONFIG environment variable
+        3. Current directory and parent directories
+        """
+        # Check explicit path
+        if self._config_path:
+            if os.path.exists(self._config_path):
+                return self._config_path
+            logger.warning(f"Config file not found: {self._config_path}")
+
+        # Check environment variable
+        env_config = os.environ.get("FLOWYML_CONFIG")
+        if env_config and os.path.exists(env_config):
+            return env_config
+
+        # Search current directory and parents
+        current = Path.cwd()
+        for _ in range(5):  # Search up to 5 levels
+            for name in CONFIG_FILE_NAMES:
+                config_file = current / name
+                if config_file.exists():
+                    return str(config_file)
+            parent = current.parent
+            if parent == current:
+                break
+            current = parent
+
+        return None
+
+    def reload(self) -> None:
+        """Reload configuration from file."""
+        self._instances.clear()
+        self._load_config()
+
+    @property
+    def plugins_config(self) -> dict:
+        """Get the plugins configuration section."""
+        return self._config.get("plugins", {})
+
+    def get_plugin_config(self, plugin_role: str) -> dict | None:
+        """Get configuration for a specific plugin role.
+
+        Args:
+            plugin_role: Role like 'experiment_tracker', 'artifact_store', etc.
+
+        Returns:
+            Configuration dictionary or None.
+        """
+        return self.plugins_config.get(plugin_role)
+
+    def _get_plugin(self, plugin_role: str, plugin_class: type = None) -> BasePlugin | None:
+        """Get or create a plugin instance for a role.
+
+        Args:
+            plugin_role: Role like 'experiment_tracker'.
+            plugin_class: Optional base class to validate against.
+
+        Returns:
+            Plugin instance or None if not configured.
+        """
+        # Check cache
+        if plugin_role in self._instances:
+            return self._instances[plugin_role]
+
+        # Get config
+        config = self.get_plugin_config(plugin_role)
+        if not config:
+            return None
+
+        # Extract plugin type
+        plugin_type = config.pop("type", None)
+        if not plugin_type:
+            logger.error(f"Plugin config for '{plugin_role}' missing 'type'")
+            return None
+
+        # Check if installed
+        if not self._manager.is_installed(plugin_type):
+            logger.info(f"Installing plugin '{plugin_type}'...")
+            self._manager.install(plugin_type)
+
+        # Load and instantiate
+        try:
+            plugin = self._manager.get_instance(plugin_type, **config)
+            plugin.initialize()
+            self._instances[plugin_role] = plugin
+            return plugin
+        except Exception as e:
+            logger.error(f"Failed to create plugin '{plugin_type}': {e}")
+            return None
+
+    def get_experiment_tracker(self) -> ExperimentTracker | None:
+        """Get the configured experiment tracker."""
+        return self._get_plugin("experiment_tracker", ExperimentTracker)
+
+    def get_artifact_store(self) -> ArtifactStorePlugin | None:
+        """Get the configured artifact store."""
+        return self._get_plugin("artifact_store", ArtifactStorePlugin)
+
+    def get_orchestrator(self) -> OrchestratorPlugin | None:
+        """Get the configured orchestrator."""
+        return self._get_plugin("orchestrator", OrchestratorPlugin)
+
+    def get_container_registry(self) -> ContainerRegistryPlugin | None:
+        """Get the configured container registry."""
+        return self._get_plugin("container_registry", ContainerRegistryPlugin)
+
+    def get_feature_store(self) -> FeatureStorePlugin | None:
+        """Get the configured feature store."""
+        return self._get_plugin("feature_store", FeatureStorePlugin)
+
+    def get_data_validator(self) -> DataValidatorPlugin | None:
+        """Get the configured data validator."""
+        return self._get_plugin("data_validator", DataValidatorPlugin)
+
+    def get_alerter(self) -> AlerterPlugin | None:
+        """Get the configured alerter."""
+        return self._get_plugin("alerter", AlerterPlugin)
+
+
+# =============================================================================
+# GLOBAL CONFIG INSTANCE
+# =============================================================================
+
+_config: PluginConfig | None = None
+
+
+def get_config(config_path: str = None) -> PluginConfig:
+    """Get the global plugin configuration.
+
+    Args:
+        config_path: Optional path to config file.
+
+    Returns:
+        PluginConfig instance.
+    """
+    global _config
+    if _config is None or config_path:
+        _config = PluginConfig(config_path)
+    return _config
+
+
+def reload_config() -> None:
+    """Reload the global configuration from file."""
+    global _config
+    if _config:
+        _config.reload()
+
+
+# =============================================================================
+# CONVENIENCE FUNCTIONS
+# =============================================================================
+
+
+def get_tracker() -> ExperimentTracker | None:
+    """Get the configured experiment tracker.
+
+    Reads configuration from flowyml.yaml and returns the configured
+    experiment tracker, ready to use.
+
+    Example:
+        tracker = get_tracker()
+        if tracker:
+            tracker.start_run("my_experiment")
+    """
+    return get_config().get_experiment_tracker()
+
+
+def get_artifact_store() -> ArtifactStorePlugin | None:
+    """Get the configured artifact store.
+
+    Example:
+        store = get_artifact_store()
+        if store:
+            store.save(model, "models/latest.pkl")
+    """
+    return get_config().get_artifact_store()
+
+
+def get_orchestrator() -> OrchestratorPlugin | None:
+    """Get the configured orchestrator.
+
+    Example:
+        orchestrator = get_orchestrator()
+        if orchestrator:
+            orchestrator.run_pipeline(my_pipeline, "run-001")
+    """
+    return get_config().get_orchestrator()
+
+
+def get_container_registry() -> ContainerRegistryPlugin | None:
+    """Get the configured container registry."""
+    return get_config().get_container_registry()
+
+
+def get_feature_store() -> FeatureStorePlugin | None:
+    """Get the configured feature store."""
+    return get_config().get_feature_store()
+
+
+def get_data_validator() -> DataValidatorPlugin | None:
+    """Get the configured data validator."""
+    return get_config().get_data_validator()
+
+
+def get_alerter() -> AlerterPlugin | None:
+    """Get the configured alerter."""
+    return get_config().get_alerter()
+
+
+# =============================================================================
+# CLI INIT COMMAND SUPPORT
+# =============================================================================
+
+
+def generate_config_template(
+    tracker: str = None,
+    store: str = None,
+    orchestrator: str = None,
+    registry: str = None,
+) -> str:
+    """Generate a flowyml.yaml template.
+
+    Args:
+        tracker: Experiment tracker plugin name.
+        store: Artifact store plugin name.
+        orchestrator: Orchestrator plugin name.
+        registry: Container registry plugin name.
+
+    Returns:
+        YAML configuration string.
+    """
+    config = {
+        "# FlowyML Configuration": None,
+        "# Run 'flowyml plugin list' to see available plugins": None,
+        "plugins": {},
+    }
+
+    if tracker:
+        config["plugins"]["experiment_tracker"] = {
+            "type": tracker,
+            "# Add plugin-specific configuration below": None,
+        }
+        if tracker == "mlflow":
+            config["plugins"]["experiment_tracker"].update(
+                {
+                    "tracking_uri": "http://localhost:5000",
+                    "experiment_name": "my_experiments",
+                },
+            )
+
+    if store:
+        config["plugins"]["artifact_store"] = {
+            "type": store,
+        }
+        if store == "s3":
+            config["plugins"]["artifact_store"].update(
+                {
+                    "bucket": "my-ml-artifacts",
+                    "prefix": "experiments/",
+                },
+            )
+        elif store == "gcs":
+            config["plugins"]["artifact_store"].update(
+                {
+                    "bucket": "my-ml-artifacts",
+                    "prefix": "experiments/",
+                    "project": "my-gcp-project",
+                },
+            )
+
+    if orchestrator:
+        config["plugins"]["orchestrator"] = {
+            "type": orchestrator,
+        }
+        if orchestrator == "vertex_ai":
+            config["plugins"]["orchestrator"].update(
+                {
+                    "project": "my-gcp-project",
+                    "location": "us-central1",
+                    "staging_bucket": "gs://my-staging-bucket",
+                },
+            )
+        elif orchestrator == "kubernetes":
+            config["plugins"]["orchestrator"].update(
+                {
+                    "namespace": "ml-pipelines",
+                },
+            )
+
+    if registry:
+        config["plugins"]["container_registry"] = {
+            "type": registry,
+        }
+        if registry == "gcr":
+            config["plugins"]["container_registry"].update(
+                {
+                    "project": "my-gcp-project",
+                    "location": "us-central1",
+                    "repository": "ml-images",
+                    "use_artifact_registry": True,
+                },
+            )
+
+    # Generate YAML
+    lines = ["# FlowyML Configuration", "# Run 'flowyml plugin list' to see available plugins", ""]
+    lines.append(yaml.dump({"plugins": config["plugins"]}, default_flow_style=False, sort_keys=False))
+
+    return "\n".join(lines)

flowyml/plugins/deployers/__init__.py

@@ -0,0 +1,22 @@
+"""FlowyML Model Deployer Plugins - Native implementations for model deployment.
+
+This module provides model deployer plugin implementations for:
+- Vertex AI Endpoints (GCP)
+- SageMaker Endpoints (AWS)
+
+Usage:
+    from flowyml.plugins.deployers import VertexEndpointDeployer, SageMakerEndpointDeployer
+
+    # Or via config
+    # model_deployer:
+    #   type: vertex_endpoint
+    #   project: my-gcp-project
+"""
+
+from flowyml.plugins.deployers.vertex import VertexEndpointDeployer
+from flowyml.plugins.deployers.sagemaker import SageMakerEndpointDeployer
+
+__all__ = [
+    "VertexEndpointDeployer",
+    "SageMakerEndpointDeployer",
+]

flowyml/plugins/deployers/gcp_cloud_run.py

@@ -0,0 +1,200 @@
+"""GCP Cloud Run Deployer - Native FlowyML Plugin.
+
+This plugin provides direct integration with Google Cloud Run
+for serverless model serving.
+"""
+
+import logging
+import subprocess
+import json
+from typing import Any
+from flowyml.utils.observability import trace_execution
+
+from flowyml.plugins.base import ModelDeployerPlugin, PluginMetadata, PluginType
+
+logger = logging.getLogger(__name__)
+
+
+# import removed from here
+
+
+class GCPCloudRunDeployer(ModelDeployerPlugin):
+    """Native GCP Cloud Run deployer for FlowyML.
+
+    This plugin deploys containerized models to Cloud Run.
+
+    Args:
+        project_id: GCP project ID.
+        region: GCP region (default: us-central1).
+    """
+
+    metadata = PluginMetadata(
+        name="gcp_cloud_run",
+        version="1.0.0",
+        description="Google Cloud Run Deployer",
+        author="FlowyML Team",
+        plugin_type=PluginType.MODEL_DEPLOYER,
+    )
+
+    def __init__(
+        self,
+        project_id: str,
+        region: str = "us-central1",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.project_id = project_id
+        self.region = region
+
+    @property
+    def plugin_type(self) -> PluginType:
+        return PluginType.MODEL_DEPLOYER
+
+    def initialize(self) -> None:
+        """Verify gcloud is installed."""
+        try:
+            subprocess.run(["gcloud", "--version"], check=True, capture_output=True)
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            raise ImportError(
+                "gcloud CLI is required for GCP Cloud Run deployment. " "Please install the Google Cloud SDK.",
+            )
+
+    @trace_execution(operation_name="cloud_run_deploy")
+    def deploy(
+        self,
+        model_uri: str,
+        endpoint_name: str,
+        image: str = None,
+        memory: str = "512Mi",
+        cpu: str = "1",
+        min_instances: int = 0,
+        max_instances: int = 10,
+        allow_unauthenticated: bool = True,
+        env_vars: dict[str, str] = None,
+        **kwargs,
+    ) -> str:
+        """Deploy a container to Cloud Run.
+
+        Args:
+            model_uri: Not used directly for Cloud Run (uses image), but kept for interface compatibility.
+            endpoint_name: Name of the Cloud Run service.
+            image: Docker image to deploy (required).
+            memory: Memory limit (e.g., 512Mi, 2Gi).
+            cpu: CPU limit.
+            min_instances: Minimum number of instances.
+            max_instances: Maximum number of instances.
+            allow_unauthenticated: Allow public access.
+            env_vars: Environment variables.
+            **kwargs: Additional arguments.
+
+        Returns:
+            Service URL.
+        """
+        if not image:
+            raise ValueError("Docker image is required for Cloud Run deployment.")
+
+        command = [
+            "gcloud",
+            "run",
+            "deploy",
+            endpoint_name,
+            f"--image={image}",
+            f"--region={self.region}",
+            f"--project={self.project_id}",
+            f"--memory={memory}",
+            f"--cpu={cpu}",
+            f"--min-instances={min_instances}",
+            f"--max-instances={max_instances}",
+            "--platform=managed",
+        ]
+
+        if allow_unauthenticated:
+            command.append("--allow-unauthenticated")
+        else:
+            command.append("--no-allow-unauthenticated")
+
+        if env_vars:
+            env_list = [f"{k}={v}" for k, v in env_vars.items()]
+            command.append(f"--set-env-vars={','.join(env_list)}")
+
+        if model_uri:
+            # Pass model URI as environment variable if provided
+            command.append(f"--set-env-vars=MODEL_URI={model_uri}")
+
+        try:
+            logger.info(f"Deploying Cloud Run service: {endpoint_name}...")
+            subprocess.run(command, check=True)
+
+            # Get service URL
+            url_cmd = [
+                "gcloud",
+                "run",
+                "services",
+                "describe",
+                endpoint_name,
+                f"--region={self.region}",
+                f"--project={self.project_id}",
+                "--format=value(status.url)",
+            ]
+            result = subprocess.run(url_cmd, check=True, capture_output=True, text=True)
+            url = result.stdout.strip()
+
+            logger.info(f"Deployed successfully to: {url}")
+            return url
+
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to deploy to Cloud Run: {e}")
+            raise
+
+    def get_endpoint(self, endpoint_name: str) -> dict | None:
+        """Get Cloud Run service details."""
+        try:
+            cmd = [
+                "gcloud",
+                "run",
+                "services",
+                "describe",
+                endpoint_name,
+                f"--region={self.region}",
+                f"--project={self.project_id}",
+                "--format=json",
+            ]
+            result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+            return json.loads(result.stdout)
+        except subprocess.CalledProcessError:
+            return None
+
+    @trace_execution(operation_name="cloud_run_undeploy")
+    def undeploy(self, endpoint_name: str) -> bool:
+        """Delete Cloud Run service."""
+        try:
+            cmd = [
+                "gcloud",
+                "run",
+                "services",
+                "delete",
+                endpoint_name,
+                f"--region={self.region}",
+                f"--project={self.project_id}",
+                "--quiet",
+            ]
+            subprocess.run(cmd, check=True)
+            logger.info(f"Deleted Cloud Run service: {endpoint_name}")
+            return True
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Failed to delete service: {e}")
+            return False
+
+    @trace_execution(operation_name="cloud_run_predict")
+    def predict(self, endpoint: str, data: Any) -> Any:
+        """Make prediction (helper using curl/requests if desired, but usually done via HTTP client)."""
+        # Simple implementation using requests if available, or just printing instructions
+        import requests
+
+        if isinstance(data, (dict, list)):
+            resp = requests.post(endpoint, json=data)
+        else:
+            resp = requests.post(endpoint, data=data)
+
+        resp.raise_for_status()
+        return resp.json()