flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/stacks/aws.py

@@ -290,6 +290,45 @@ class AWSBatchOrchestrator(RemoteOrchestrator):
             print(f"Error fetching job status: {e}")
             return ExecutionStatus.FAILED
 
+    def get_run_logs(self, job_id: str) -> str:
+        """Get logs for an AWS Batch job.
+
+        Args:
+            job_id: The job ID.
+
+        Returns:
+            String containing the logs.
+        """
+        import boto3
+
+        try:
+            client = self._client()
+            response = client.describe_jobs(jobs=[job_id])
+
+            if not response.get("jobs"):
+                return "Job not found."
+
+            job = response["jobs"][0]
+            if "container" not in job or "logStreamName" not in job["container"]:
+                # Job might not have started yet or failed before logging
+                return "Logs not available yet (no log stream)."
+
+            log_stream_name = job["container"]["logStreamName"]
+            logs_client = boto3.client("logs", region_name=self.region)
+
+            log_events = logs_client.get_log_events(
+                logGroupName="/aws/batch/job",
+                logStreamName=log_stream_name,
+                limit=100,
+                startFromHead=False,
+            )
+
+            messages = [event["message"] for event in log_events.get("events", [])]
+            return "\n".join(messages) if messages else "No log messages found."
+
+        except Exception as e:
+            return f"Failed to fetch logs: {e}"
+
     def stop_run(self, job_id: str, graceful: bool = True) -> None:
         """Stop an AWS Batch job.
 
@@ -516,6 +555,52 @@ class SageMakerOrchestrator(RemoteOrchestrator):
             print(f"Error fetching training job status: {e}")
             return ExecutionStatus.FAILED
 
+    def get_run_logs(self, job_id: str) -> str:
+        """Get logs for a SageMaker training job.
+
+        Args:
+            job_id: The training job name.
+
+        Returns:
+            String containing the logs.
+        """
+        import boto3
+
+        try:
+            self._client()
+            logs_client = boto3.client("logs", region_name=self.region)
+
+            # SageMaker logs to /aws/sagemaker/TrainingJobs
+            log_group_name = "/aws/sagemaker/TrainingJobs"
+
+            # Find log stream
+            # Streams are usually like <job_name>/algo-1-123456789
+            streams = logs_client.describe_log_streams(
+                logGroupName=log_group_name,
+                logStreamNamePrefix=job_id,
+                orderBy="LastEventTime",
+                descending=True,
+                limit=1,
+            )
+
+            if not streams.get("logStreams"):
+                return "No log streams found for this job."
+
+            log_stream_name = streams["logStreams"][0]["logStreamName"]
+
+            log_events = logs_client.get_log_events(
+                logGroupName=log_group_name,
+                logStreamName=log_stream_name,
+                limit=100,
+                startFromHead=False,
+            )
+
+            messages = [event["message"] for event in log_events.get("events", [])]
+            return "\n".join(messages) if messages else "No log messages found."
+
+        except Exception as e:
+            return f"Failed to fetch logs: {e}"
+
     def stop_run(self, job_id: str, graceful: bool = True) -> None:
         """Stop a SageMaker training job.
 
@@ -556,6 +641,7 @@ class AWSStack(Stack):
         orchestrator_type: str = "batch",
         role_arn: str | None = None,
         metadata_store: Any | None = None,
+        model_deployer: Any | None = None,
     ):
         orchestrator: Orchestrator
         if orchestrator_type == "sagemaker":
@@ -569,6 +655,11 @@ class AWSStack(Stack):
         if metadata_store is None:
             metadata_store = SQLiteMetadataStore()
 
+        if model_deployer is None:
+            from flowyml.plugins.deployers.sagemaker import SageMakerEndpointDeployer
+
+            model_deployer = SageMakerEndpointDeployer(region=region, role_arn=role_arn)
+
         super().__init__(
             name=name,
             executor=None,
@@ -576,6 +667,7 @@ class AWSStack(Stack):
             metadata_store=metadata_store,
             container_registry=container_registry,
             orchestrator=orchestrator,
+            model_deployer=model_deployer,
         )
 
         self.region = region
@@ -596,4 +688,5 @@ class AWSStack(Stack):
             "orchestrator": self.orchestrator.to_dict(),
             "artifact_store": self.artifact_store.to_dict(),
             "container_registry": self.container_registry.to_dict(),
+            "model_deployer": self.model_deployer.to_dict() if self.model_deployer else None,
         }

flowyml/stacks/base.py

@@ -14,6 +14,7 @@ class StackConfig:
     metadata_store: str
     container_registry: str | None = None
     orchestrator: str | None = None
+    model_deployer: str | None = None
 
     def to_dict(self) -> dict[str, Any]:
         """Convert to dictionary."""
@@ -24,6 +25,7 @@ class StackConfig:
             "metadata_store": self.metadata_store,
             "container_registry": self.container_registry,
             "orchestrator": self.orchestrator,
+            "model_deployer": self.model_deployer,
         }
 
 
@@ -46,6 +48,7 @@ class Stack:
         metadata_store: Any,
         container_registry: Any | None = None,
         orchestrator: Any | None = None,
+        model_deployer: Any | None = None,
     ):
         self.name = name
         self.executor = executor
@@ -53,6 +56,7 @@ class Stack:
         self.metadata_store = metadata_store
         self.container_registry = container_registry
         self.orchestrator = orchestrator
+        self.model_deployer = model_deployer
 
         self.config = StackConfig(
             name=name,
@@ -61,6 +65,7 @@ class Stack:
             metadata_store=type(metadata_store).__name__,
             container_registry=type(container_registry).__name__ if container_registry else None,
             orchestrator=type(orchestrator).__name__ if orchestrator else None,
+            model_deployer=type(model_deployer).__name__ if model_deployer else None,
         )
 
     def activate(self) -> None:
@@ -68,6 +73,63 @@ class Stack:
         # In a real implementation, this would set the global active stack
         pass
 
+    def prepare_docker_image(self, docker_config: Any, pipeline_name: str, project_name: str | None = None) -> str:
+        """Prepare the Docker image for execution.
+
+        Args:
+            docker_config: Docker configuration object.
+            pipeline_name: Name of the pipeline being built.
+            project_name: Optional name of the project.
+
+        Returns:
+            str: The full URI of the docker image to use.
+
+        Raises:
+            ValueError: If image cannot be prepared (e.g. no registry configured for build).
+        """
+        # 1. If explicit image provided, use it
+        if docker_config.image:
+            return docker_config.image
+
+        # 2. If no registry, we cannot build/push for remote execution
+        if not self.container_registry:
+            raise ValueError(
+                "Remote execution requires a specific 'image' in DockerConfiguration "
+                "or a configured 'container_registry' in the Stack for automatic building.",
+            )
+
+        # 3. Trigger build and push
+        # Use safe naming: registry/project/pipeline:latest OR registry/pipeline:latest
+        if project_name:
+            image_name = f"{project_name}-{pipeline_name}"
+        else:
+            image_name = pipeline_name
+
+        # Clean image name to be docker compatible (lowercase, alphanumeric)
+        import re
+
+        safe_name = re.sub(r"[^a-zA-Z0-9_\-]", "_", image_name).lower()
+
+        image_tag = f"{self.container_registry.registry_uri}/{safe_name}:latest"
+
+        # Build
+        try:
+            from flowyml.core.image_builder import DockerImageBuilder
+
+            builder = DockerImageBuilder()
+            builder.build_image(docker_config, image_tag)
+        except ImportError:
+            # Fallback if file not found (shouldn't happen in prod)
+            print("Warning: DockerImageBuilder not found. Skipping build.")
+
+        # Push
+        print(f"🚀 Pushing image: {image_tag}")
+        try:
+            pushed_uri = self.container_registry.push_image(image_tag)
+            return pushed_uri
+        except Exception as e:
+            raise RuntimeError(f"Failed to push image to registry: {e}")
+
     def validate(self) -> bool:
         """Validate that all stack components are properly configured."""
         # Check that all components are properly configured

flowyml/stacks/components.py

@@ -18,6 +18,7 @@ class ComponentType(Enum):
     CONTAINER_REGISTRY = "container_registry"
     METADATA_STORE = "metadata_store"
     EXECUTOR = "executor"
+    MODEL_DEPLOYER = "model_deployer"
 
 
 @dataclass
@@ -130,6 +131,17 @@ class Orchestrator(StackComponent):
         """Get status of a pipeline run."""
         pass
 
+    def get_run_logs(self, run_id: str) -> str:
+        """Get logs for a pipeline run.
+
+        Args:
+            run_id: The run identifier.
+
+        Returns:
+            String containing the logs.
+        """
+        return "Logs not available for this orchestrator."
+
 
 class ArtifactStore(StackComponent):
     """Base class for artifact stores."""

flowyml/stacks/gcp.py

@@ -41,7 +41,7 @@ class VertexAIOrchestrator(RemoteOrchestrator):
         self,
         name: str = "vertex_ai",
         project_id: str | None = None,
-        region: str = "us-central1",
+        region: str = "europe-west1",
         service_account: str | None = None,
         network: str | None = None,
         encryption_key: str | None = None,
@@ -176,6 +176,36 @@ class VertexAIOrchestrator(RemoteOrchestrator):
             print(f"Error fetching job status: {e}")
             return ExecutionStatus.FAILED
 
+    def get_run_logs(self, job_id: str) -> str:
+        """Get logs for a Vertex AI job.
+
+        Args:
+            job_id: The job resource name.
+
+        Returns:
+            String containing the logs.
+        """
+        try:
+            from google.cloud import logging
+
+            client = logging.Client(project=self.project_id)
+            job_name = job_id.split("/")[-1]
+
+            # Filter logs for this job
+            # Note: This is a simplified filter; exact filter depends on Vertex AI logging format
+            filter_str = f'resource.type="ml_job" AND conversion_id="{job_name}"'
+
+            entries = client.list_entries(filter_=filter_str, order_by=logging.DESCENDING, max_results=100)
+            logs = []
+            for entry in entries:
+                if entry.payload:
+                    logs.append(str(entry.payload))
+
+            return "\n".join(reversed(logs)) if logs else "No logs found."
+
+        except Exception as e:
+            return f"Failed to fetch logs: {e}"
+
     def stop_run(self, job_id: str, graceful: bool = True) -> None:
         """Cancel a Vertex AI job."""
         from google.cloud import aiplatform
@@ -489,11 +519,12 @@ class GCPStack(Stack):
         self,
         name: str = "gcp",
         project_id: str | None = None,
-        region: str = "us-central1",
+        region: str = "europe-west1",
         bucket_name: str | None = None,
         registry_uri: str | None = None,
         service_account: str | None = None,
         metadata_store: Any | None = None,
+        model_deployer: Any | None = None,
     ):
         """Initialize GCP stack.
 
@@ -505,6 +536,7 @@ class GCPStack(Stack):
             registry_uri: Container registry URI
             service_account: Service account for job execution
             metadata_store: Metadata store (optional, defaults to local SQLite)
+            model_deployer: Optional model deployer
         """
         # Create GCP components
         orchestrator = VertexAIOrchestrator(
@@ -524,11 +556,12 @@ class GCPStack(Stack):
             region=region,
         )
 
-        # Use local metadata store if not provided
-        if metadata_store is None:
-            from flowyml.storage.metadata import SQLiteMetadataStore
+        # Use new generic deployer if provided, else use CloudRun default if desired,
+        # but better to stick to generic injection or default creation
+        if model_deployer is None:
+            from flowyml.plugins.deployers.gcp_cloud_run import GCPCloudRunDeployer
 
-            metadata_store = SQLiteMetadataStore()
+            model_deployer = GCPCloudRunDeployer(project_id=project_id, region=region)
 
         # Initialize base stack
         super().__init__(
@@ -538,12 +571,13 @@ class GCPStack(Stack):
             metadata_store=metadata_store,
             container_registry=container_registry,
             orchestrator=orchestrator,
+            model_deployer=model_deployer,
         )
 
         self.project_id = project_id
         self.region = region
+        # Legacy helpers kept for backward compatibility if needed, but stack now uses proper components
         self.vertex_endpoints = VertexEndpointManager(project_id=project_id, region=region)
-        self.cloud_run = CloudRunDeployer(project_id=project_id, region=region)
 
     def validate(self) -> bool:
         """Validate all GCP stack components."""
@@ -562,13 +596,14 @@ class GCPStack(Stack):
             "orchestrator": self.orchestrator.to_dict(),
             "artifact_store": self.artifact_store.to_dict(),
             "container_registry": self.container_registry.to_dict(),
+            "model_deployer": self.model_deployer.to_dict() if self.model_deployer else None,
         }
 
 
 class VertexEndpointManager:
     """Deploy trained models as Vertex AI endpoints."""
 
-    def __init__(self, project_id: str | None, region: str = "us-central1"):
+    def __init__(self, project_id: str | None, region: str = "europe-west1"):
         self.project_id = project_id
         self.region = region
 
@@ -600,7 +635,7 @@ class VertexEndpointManager:
 class CloudRunDeployer:
     """Deploy container images to Cloud Run."""
 
-    def __init__(self, project_id: str | None, region: str = "us-central1"):
+    def __init__(self, project_id: str | None, region: str = "europe-west1"):
         self.project_id = project_id
         self.region = region
 

flowyml/stacks/plugins.py

@@ -26,6 +26,14 @@ from flowyml.stacks.components import (
 from flowyml.stacks.bridge import GenericBridge, AdaptationRule
 
 
+# Lazy import to avoid circular dependencies
+def _get_zenml_bridge():
+    """Lazy import of ZenMLBridge to avoid import errors when ZenML is not installed."""
+    from flowyml.stacks.zenml_bridge import ZenMLBridge
+
+    return ZenMLBridge()
+
+
 @dataclass
 class PluginInfo:
     """Metadata about a plugin."""
@@ -334,6 +342,113 @@ class ComponentRegistry:
         except subprocess.CalledProcessError:
             return False
 
+    # ==================== ZenML Integration Methods ====================
+
+    def list_zenml_integrations(self) -> list[str]:
+        """List all available ZenML integrations.
+
+        Returns:
+            List of integration names (e.g., ['mlflow', 'kubernetes', 'aws']).
+        """
+        try:
+            bridge = _get_zenml_bridge()
+            return bridge.list_available_integrations()
+        except Exception:
+            return []
+
+    def list_installed_zenml_integrations(self) -> list[str]:
+        """List installed ZenML integrations.
+
+        Returns:
+            List of installed integration names.
+        """
+        try:
+            bridge = _get_zenml_bridge()
+            return bridge.list_installed_integrations()
+        except Exception:
+            return []
+
+    def install_zenml_integration(self, integration_name: str) -> bool:
+        """Install a ZenML integration and its dependencies.
+
+        Args:
+            integration_name: Name of the integration (e.g., "mlflow", "kubernetes").
+
+        Returns:
+            True if installation was successful.
+
+        Example:
+            >>> registry = get_component_registry()
+            >>> registry.install_zenml_integration("mlflow")
+            True
+        """
+        try:
+            bridge = _get_zenml_bridge()
+            return bridge.install_integration(integration_name)
+        except Exception as e:
+            print(f"Failed to install ZenML integration: {e}")
+            return False
+
+    def import_zenml_integration(self, integration_name: str) -> list[type[StackComponent]]:
+        """Import all components from a ZenML integration.
+
+        This discovers all flavors provided by a ZenML integration and
+        registers them as FlowyML components.
+
+        Args:
+            integration_name: Name of the integration to import.
+
+        Returns:
+            List of wrapped FlowyML component classes.
+
+        Example:
+            >>> registry = get_component_registry()
+            >>> components = registry.import_zenml_integration("mlflow")
+            >>> print([c.__name__ for c in components])
+            ['ZenMLMLFlowExperimentTrackerWrapper']
+        """
+        try:
+            bridge = _get_zenml_bridge()
+            components = bridge.import_integration(integration_name)
+
+            # Register all imported components
+            for component_class in components:
+                self.register(component_class)
+
+            return components
+        except Exception as e:
+            print(f"Failed to import ZenML integration: {e}")
+            return []
+
+    def import_all_zenml(self) -> dict[str, list[type[StackComponent]]]:
+        """Import all components from all installed ZenML integrations.
+
+        This is the easiest way to make all ZenML components available
+        in FlowyML with a single call.
+
+        Returns:
+            Dictionary mapping integration names to lists of wrapped components.
+
+        Example:
+            >>> registry = get_component_registry()
+            >>> all_components = registry.import_all_zenml()
+            >>> print(all_components.keys())
+            dict_keys(['mlflow', 'kubernetes', 'aws'])
+        """
+        try:
+            bridge = _get_zenml_bridge()
+            result = bridge.import_all()
+
+            # Register all imported components
+            for _integration_name, components in result.items():
+                for component_class in components:
+                    self.register(component_class)
+
+            return result
+        except Exception as e:
+            print(f"Failed to import ZenML integrations: {e}")
+            return {}
+
     @staticmethod
     def _class_to_snake_case(name: str) -> str:
         """Convert ClassName to class_name."""

flowyml/stacks/registry.py

@@ -226,7 +226,8 @@ def get_active_stack() -> Stack | None:
         from flowyml.stacks.base import Stack
         import os
 
-        api_token = os.getenv("FLOWYML_API_TOKEN")
+        # Prefer token from config, fallback to env var
+        api_token = config.api_token or os.getenv("FLOWYML_API_TOKEN")
 
         return Stack(
             name="remote_logging",