flowyml 1.7.2__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/cli/stack_cli.py

@@ -98,23 +98,28 @@ def stack() -> None:
 @click.option("--config", "-c", help="Path to flowyml.yaml")
 def list_stacks(config: str | None) -> None:
     """List all configured stacks."""
-    from flowyml.utils.stack_config import load_config
+    from flowyml.plugins.stack_config import get_stack_manager
+    from flowyml.plugins.config import get_config
 
-    loader = load_config(config)
-    stacks = loader.list_stacks()
+    # Initialize config if path provided
+    if config:
+        get_config(config)
+
+    manager = get_stack_manager()
+    stacks = manager.list_stacks()
 
     if not stacks:
-        click.echo("No stacks configured. Create a flowyml.yaml file.")
+        click.echo("No stacks configured. Create a flowyml.yaml file with 'stacks:' section.")
         return
 
-    default = loader.get_default_stack()
+    active = manager.active_stack_name
 
-    click.echo("\nConfigured stacks:")
+    click.echo("\n📦 Configured stacks:")
     for stack_name in stacks:
-        marker = " (default)" if stack_name == default else ""
-        config_data = loader.get_stack_config(stack_name)
-        stack_type = config_data.get("type", "unknown")
-        click.echo(f"  • {stack_name}{marker} [{stack_type}]")
+        marker = " ✓ (active)" if stack_name == active else ""
+        stack = manager.get_stack(stack_name)
+        orch_type = stack.orchestrator.get("type", "local") if stack and stack.orchestrator else "local"
+        click.echo(f"  • {stack_name}{marker} [orchestrator: {orch_type}]")
     click.echo()
 
 
@@ -123,35 +128,143 @@ def list_stacks(config: str | None) -> None:
 @click.option("--config", "-c", help="Path to flowyml.yaml")
 def show_stack(stack_name: str, config: str | None) -> None:
     """Show detailed stack configuration."""
-    from flowyml.utils.stack_config import load_config
-    import yaml
+    from flowyml.plugins.stack_config import get_stack_manager
+    from flowyml.plugins.config import get_config
 
-    loader = load_config(config)
-    stack_config = loader.get_stack_config(stack_name)
+    # Initialize config if path provided
+    if config:
+        get_config(config)
 
-    if not stack_config:
+    manager = get_stack_manager()
+    stack = manager.get_stack(stack_name)
+
+    if not stack:
         click.echo(f"Stack '{stack_name}' not found", err=True)
+        available = manager.list_stacks()
+        if available:
+            click.echo(f"Available stacks: {', '.join(available)}")
         sys.exit(1)
 
-    click.echo(f"\nStack: {stack_name}")
-    click.echo(yaml.dump(stack_config, default_flow_style=False))
+    is_active = stack_name == manager.active_stack_name
+    status = " (active)" if is_active else ""
+
+    click.echo(f"\n📦 Stack: {stack_name}{status}")
+    click.echo("─" * 40)
+
+    # Show components
+    if stack.orchestrator:
+        click.echo(f"\n🎯 Orchestrator: {stack.orchestrator.get('type', 'unknown')}")
+        for k, v in stack.orchestrator.items():
+            if k != "type":
+                click.echo(f"   {k}: {v}")
+
+    if stack.artifact_store:
+        click.echo(f"\n💾 Artifact Store: {stack.artifact_store.get('type', 'unknown')}")
+        for k, v in stack.artifact_store.items():
+            if k != "type":
+                click.echo(f"   {k}: {v}")
+
+    if stack.experiment_tracker:
+        click.echo(f"\n📊 Experiment Tracker: {stack.experiment_tracker.get('type', 'unknown')}")
+
+    if stack.model_registry:
+        click.echo(f"\n📝 Model Registry: {stack.model_registry.get('type', 'unknown')}")
+
+    if stack.model_deployer:
+        click.echo(f"\n🚀 Model Deployer: {stack.model_deployer.get('type', 'unknown')}")
+
+    if stack.container_registry:
+        click.echo(f"\n🐳 Container Registry: {stack.container_registry.get('type', 'unknown')}")
+
+    if stack.artifact_routing:
+        click.echo("\n📍 Artifact Routing:")
+        for type_name, rule in stack.artifact_routing.rules.items():
+            click.echo(f"   {type_name}: store={rule.store}, register={rule.register}")
+
+    click.echo()
 
 
 @stack.command("set-default")
 @click.argument("stack_name")
 @click.option("--config", "-c", help="Path to flowyml.yaml")
-def set_default_stack(stack_name: str, config: str | None) -> None:
-    """Set the default stack."""
-    from flowyml.stacks.registry import get_registry
+def set_active_stack(stack_name: str, config: str | None) -> None:
+    """Set the active stack (alias for switch)."""
+    switch_stack_impl(stack_name, config)
 
-    registry = get_registry()
 
-    if stack_name not in registry.list_stacks():
-        click.echo(f"Stack '{stack_name}' not found", err=True)
+@stack.command("switch")
+@click.argument("stack_name")
+@click.option("--config", "-c", help="Path to flowyml.yaml")
+@click.option("--validate/--no-validate", default=True, help="Validate stack configuration after switching")
+def switch_stack(stack_name: str, config: str | None, validate: bool) -> None:
+    """Switch to a different stack."""
+    switch_stack_impl(stack_name, config, validate)
+
+
+def switch_stack_impl(stack_name: str, config: str | None, validate: bool = False) -> None:
+    from flowyml.plugins.stack_config import get_stack_manager
+    from flowyml.plugins.config import get_config
+
+    from rich.console import Console
+
+    console = Console()
+
+    # Initialize config if path provided
+    if config:
+        get_config(config)
+
+    manager = get_stack_manager()
+
+    if manager.set_active_stack(stack_name):
+        console.print(f"[bold green]✅ Active stack set to '{stack_name}'[/bold green]")
+
+        if validate:
+            stack = manager.get_stack(stack_name)
+            console.print(f"🔍 Validating stack '{stack_name}' environment...")
+
+            # Check for remote requirements
+            if stack.orchestrator and stack.orchestrator.get("type") != "local":
+                console.print("[yellow]Remote stack detected. Checking Docker configuration...[/yellow]")
+                if not stack.container_registry:
+                    console.print(
+                        "[bold red]❌ Warning: Remote stack usually requires a Container Registry for automatic builds.[/bold red]",
+                    )
+                    console.print(
+                        "   Please configure 'container_registry' in your stack or ensure you provide pre-built image URIs.",
+                    )
+    else:
+        console.print(f"[bold red]❌ Stack '{stack_name}' not found[/bold red]")
+        available = manager.list_stacks()
+        if available:
+            console.print(f"Available stacks: {', '.join(available)}")
+        sys.exit(1)
+
+
+@stack.command("register")
+@click.argument("stack_name")
+@click.option("--file", "-f", "config_file", required=True, help="Path to stack config file")
+def register_stack(stack_name: str, config_file: str) -> None:
+    """Register a new stack from a config file."""
+    import yaml
+    from flowyml.plugins.stack_config import get_stack_manager, StackConfig
+
+    config_path = Path(config_file)
+    if not config_path.exists():
+        click.echo(f"Config file not found: {config_file}", err=True)
         sys.exit(1)
 
-    registry.set_active_stack(stack_name)
-    click.echo(f"Set '{stack_name}' as active stack")
+    try:
+        with open(config_path) as f:
+            stack_data = yaml.safe_load(f)
+
+        stack_config = StackConfig.from_dict(stack_name, stack_data)
+        manager = get_stack_manager()
+        manager.register_stack(stack_name, stack_config)
+
+        click.echo(f"✅ Registered stack '{stack_name}' from {config_file}")
+    except Exception as e:
+        click.echo(f"❌ Error registering stack: {e}", err=True)
+        sys.exit(1)
 
 
 @cli.command()

flowyml/core/__init__.py

@@ -27,6 +27,15 @@ from flowyml.core.resources import (
     NodeAffinity,
     resources,
 )
+from flowyml.core.types import (
+    Artifact,
+    Model,
+    Dataset,
+    Metrics,
+    Parameters,
+    is_artifact_type,
+    get_artifact_type_name,
+)
 
 __all__ = [
     # Context
@@ -65,4 +74,12 @@ __all__ = [
     "GPUConfig",
     "NodeAffinity",
     "resources",
+    # Artifact Types
+    "Artifact",
+    "Model",
+    "Dataset",
+    "Metrics",
+    "Parameters",
+    "is_artifact_type",
+    "get_artifact_type_name",
 ]

flowyml/core/executor.py

@@ -12,6 +12,7 @@ import ctypes
 import requests
 import os
 import inspect
+import psutil
 
 
 class StopExecutionError(Exception):
@@ -133,10 +134,25 @@ class MonitorThread(threading.Thread):
     def run(self):
         while not self._stop_event.is_set():
             try:
-                # Send heartbeat
+                # Collect metrics
+                process = psutil.Process()
+                with process.oneshot():
+                    cpu_percent = process.cpu_percent(interval=None)
+                    memory_info = process.memory_info()
+                    memory_mb = memory_info.rss / 1024 / 1024
+
+                # Send heartbeat with metrics
                 response = requests.post(
                     f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/heartbeat",
-                    json={"step_name": self.step_name, "status": "running"},
+                    json={
+                        "step_name": self.step_name,
+                        "status": "running",
+                        "metrics": {
+                            "cpu_percent": cpu_percent,
+                            "memory_mb": memory_mb,
+                            "timestamp": datetime.now().isoformat(),
+                        },
+                    },
                     timeout=2,
                 )
                 if response.status_code == 200:
@@ -194,6 +210,10 @@ class Executor:
         inputs: dict[str, Any],
         context_params: dict[str, Any],
         cache_store: Any | None = None,
+        artifact_store: Any | None = None,
+        run_id: str | None = None,
+        project_name: str = "default",
+        all_outputs: dict[str, Any] | None = None,
     ) -> ExecutionResult:
         """Execute a single step.
 
@@ -202,6 +222,10 @@ class Executor:
             inputs: Input data for the step
             context_params: Parameters from context
             cache_store: Cache store for caching
+            artifact_store: Artifact store for logging results
+            run_id: Unique ID for this pipeline run
+            project_name: Name of the project
+            all_outputs: Collection of all step outputs for conditional evaluation
 
         Returns:
             ExecutionResult with output or error
@@ -249,21 +273,43 @@ class LocalExecutor(Executor):
         artifact_store: Any | None = None,
         run_id: str | None = None,
         project_name: str = "default",
+        all_outputs: dict[str, Any] | None = None,
     ) -> ExecutionResult:
-        """Execute step locally with retry, caching, and materialization."""
+        """Execute step locally with retry, caching, and materialization.
+
+        Args:
+            step: Step to execute
+            inputs: Input data for the step
+            context_params: Parameters from context
+            cache_store: Cache store for caching
+            artifact_store: Artifact store for logging results
+            run_id: Unique ID for this pipeline run
+            project_name: Name of the project
+            all_outputs: Collection of all step outputs for conditional evaluation
+
+        Returns:
+            ExecutionResult with output or error
+        """
         start_time = time.time()
         retries = 0
 
         # Check condition
         if step.condition:
             try:
-                # We pass inputs and context params to condition if it accepts them
-                # For simplicity, let's try to inspect the condition function
-                # or just pass what we can.
-                # A simple approach: pass nothing if it takes no args, or kwargs if it does.
-                # But inspect is safer.
+                # Prepare kwargs for condition: inputs + context_params + all_outputs
                 sig = inspect.signature(step.condition)
-                kwargs = {**inputs, **context_params}
+                kwargs = {**context_params}
+
+                # Add all outputs so far (paths like 'data/processed')
+                if all_outputs:
+                    kwargs.update(all_outputs)
+                    # Also flatten dict outputs to allow access to keys like 'quality_score'
+                    for val in all_outputs.values():
+                        if isinstance(val, dict):
+                            kwargs.update({k: v for k, v in val.items() if k not in kwargs})
+
+                # Add direct inputs (might override all_outputs if paths match)
+                kwargs.update(inputs)
 
                 # Filter kwargs to only what condition accepts
                 cond_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters}
@@ -337,7 +383,12 @@ class LocalExecutor(Executor):
                         )
                         monitor_thread.start()
 
-                    result = step.func(**kwargs)
+                    # Filter kwargs to only what the function accepts
+                    func_sig = inspect.signature(step.func)
+                    # Handle *args/**kwargs if needed, but for now strict matching is safer for steps
+                    filtered_kwargs = {k: v for k, v in kwargs.items() if k in func_sig.parameters}
+
+                    result = step.func(**filtered_kwargs)
                 except StopExecution:
                     duration = time.time() - start_time
                     return ExecutionResult(
@@ -364,16 +415,57 @@ class LocalExecutor(Executor):
                         monitor_thread.join()
 
                 # Materialize output if artifact store is available
+                # Only upload if the result is an Asset with upload=True
                 artifact_uri = None
                 if artifact_store and result is not None and run_id:
-                    with contextlib.suppress(Exception):
-                        artifact_uri = artifact_store.materialize(
-                            obj=result,
-                            name="output",  # Default name for single output
-                            run_id=run_id,
+                    # Check if result is an Asset and respects upload flag
+                    should_upload = True
+                    try:
+                        from flowyml.assets.base import Asset
+
+                        if isinstance(result, Asset):
+                            should_upload = getattr(result, "upload", False)
+                    except ImportError:
+                        pass
+
+                    if should_upload:
+                        with contextlib.suppress(Exception):
+                            artifact_uri = artifact_store.materialize(
+                                obj=result,
+                                name="output",  # Default name for single output
+                                run_id=run_id,
+                                step_name=step.name,
+                                project_name=project_name,
+                            )
+
+                # Type-based artifact routing
+                routing_result = None
+                try:
+                    from flowyml.core.routing import route_artifact, should_route
+
+                    if should_route(result):
+                        # Get return type annotation if available
+                        return_type = None
+                        try:
+                            from flowyml.core.routing import get_step_return_type
+
+                            return_type = get_step_return_type(step.func)
+                        except Exception:
+                            pass
+
+                        routing_result = route_artifact(
+                            output=result,
                             step_name=step.name,
+                            run_id=run_id or "local",
+                            return_type=return_type,
                             project_name=project_name,
                         )
+                        if routing_result and routing_result.store_uri:
+                            artifact_uri = routing_result.store_uri
+                except ImportError:
+                    pass  # Routing module not available
+                except Exception:
+                    pass  # Routing failed, continue with normal flow
 
                 # Cache result
                 if cache_store and step.cache:
@@ -574,33 +666,76 @@ class DistributedExecutor(Executor):
         inputs: dict[str, Any],
         context_params: dict[str, Any],
         cache_store: Any | None = None,
+        artifact_store: Any | None = None,
+        run_id: str | None = None,
+        project_name: str = "default",
+        all_outputs: dict[str, Any] | None = None,
     ) -> ExecutionResult:
-        """Execute step in distributed manner."""
+        """Execute step in distributed manner.
+
+        Args:
+            step: Step to execute
+            inputs: Input data for the step
+            context_params: Parameters from context
+            cache_store: Cache store for caching
+            artifact_store: Artifact store for logging results
+            run_id: Unique ID for this pipeline run
+            project_name: Name of the project
+            all_outputs: Collection of all step outputs for conditional evaluation
+
+        Returns:
+            ExecutionResult with output or error
+        """
         # Placeholder - would use Ray, Dask, or similar
         # For now, fall back to local execution
         local_executor = LocalExecutor()
-        return local_executor.execute_step(step, inputs, context_params, cache_store)
+        return local_executor.execute_step(
+            step,
+            inputs,
+            context_params,
+            cache_store,
+            artifact_store,
+            run_id,
+            project_name,
+            all_outputs,
+        )
 
     def execute_step_group(
         self,
         step_group,  # StepGroup
         inputs: dict[str, Any],
-        context_params: dict[str, Any],
+        context: Any | None = None,  # Context object for per-step injection
+        context_params: dict[str, Any] | None = None,  # Deprecated: use context instead
         cache_store: Any | None = None,
         artifact_store: Any | None = None,
         run_id: str | None = None,
         project_name: str = "default",
     ) -> list[ExecutionResult]:
-        """Execute step group in distributed manner."""
+        """Execute step group in distributed manner.
+
+        Args:
+            step_group: StepGroup to execute
+            inputs: Input data available to the group
+            context: Context object for per-step parameter injection (preferred)
+            context_params: Parameters from context (deprecated, use context instead)
+            cache_store: Cache store for caching
+            artifact_store: Artifact store for materialization
+            run_id: Run identifier
+            project_name: Project name
+
+        Returns:
+            List of ExecutionResult (one per step)
+        """
         # Placeholder - in real implementation, would send entire group to remote worker
         # For now, fall back to local execution
         local_executor = LocalExecutor()
         return local_executor.execute_step_group(
-            step_group,
-            inputs,
-            context_params,
-            cache_store,
-            artifact_store,
-            run_id,
-            project_name,
+            step_group=step_group,
+            inputs=inputs,
+            context=context,
+            context_params=context_params,
+            cache_store=cache_store,
+            artifact_store=artifact_store,
+            run_id=run_id,
+            project_name=project_name,
         )

flowyml/core/image_builder.py

@@ -0,0 +1,129 @@
+import subprocess
+from pathlib import Path
+from flowyml.stacks.components import DockerConfig
+
+
+class DockerImageBuilder:
+    """Handles building and pushing Docker images for remote execution."""
+
+    def build_image(self, docker_config: DockerConfig, tag: str) -> str:
+        """Build a Docker image from the configuration.
+
+        Args:
+            docker_config: The Docker configuration.
+            tag: The tag to apply to the built image.
+
+        Returns:
+            The full image tag that was built.
+        """
+        build_context = Path(docker_config.build_context)
+        if not build_context.exists():
+            raise FileNotFoundError(f"Build context not found: {build_context}")
+
+        # Auto-generate Dockerfile if needed
+        dockerfile_path = self._ensure_dockerfile(docker_config, build_context)
+
+        cmd = [
+            "docker",
+            "build",
+            "-t",
+            tag,
+            "-f",
+            str(dockerfile_path),
+            str(build_context),
+        ]
+
+        # Add build args
+        for k, v in docker_config.build_args.items():
+            cmd.extend(["--build-arg", f"{k}={v}"])
+
+        print(f"🐳 Building image: {tag}")
+        try:
+            subprocess.run(cmd, check=True)
+            print("✅ Build successful!")
+            return tag
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Docker build failed: {e}")
+
+    def _ensure_dockerfile(self, config: DockerConfig, context: Path) -> Path:
+        """Get path to Dockerfile or generate one."""
+        if config.dockerfile:
+            path = context / config.dockerfile
+            if not path.exists():
+                # Try absolute path
+                path = Path(config.dockerfile)
+                if not path.exists():
+                    raise FileNotFoundError(f"Dockerfile not found: {config.dockerfile}")
+            return path
+
+        # Generate temporary Dockerfile
+        generated_path = context / ".flowyml.Dockerfile"
+        content = self._generate_dockerfile_content(config)
+        generated_path.write_text(content)
+        return generated_path
+
+    def _generate_dockerfile_content(self, config: DockerConfig) -> str:
+        """Generate Dockerfile content based on requirements.
+
+        Prioritizes:
+        1. uv.lock -> uv sync
+        2. poetry.lock -> poetry install
+        3. requirements.txt -> uv pip install
+        4. list -> uv pip install
+        """
+        lines = [f"FROM {config.base_image}", "WORKDIR /app"]
+
+        # Install system dependencies if any
+        # lines.append("RUN apt-get update && apt-get install -y ...")
+
+        context_path = Path(config.build_context)
+
+        # 0. Always install uv as it's our preferred installer for pip/reqs too
+        # We install it via the official installer script for speed and isolation
+        lines.append("RUN pip install uv")
+        lines.append("ENV VIRTUAL_ENV=/app/.venv")
+        lines.append('ENV PATH="$VIRTUAL_ENV/bin:$PATH"')
+
+        # 1. Check for uv.lock
+        if (context_path / "uv.lock").exists():
+            print("📦 Detected uv based project")
+            lines.append("COPY pyproject.toml uv.lock ./")
+            # Create venv and sync
+            lines.append("RUN uv venv && uv sync --frozen --no-install-project")
+
+        # 2. Check for poetry.lock
+        elif (context_path / "poetry.lock").exists() or (context_path / "pyproject.toml").exists():
+            print("📦 Detected Poetry based project")
+            lines.append("RUN pip install poetry")
+            lines.append("COPY pyproject.toml poetry.lock* ./")
+            lines.append("RUN poetry config virtualenvs.in-project true")
+            lines.append("RUN poetry install --no-interaction --no-ansi --no-root")
+            # Add local venv to path if poetry created one
+            lines.append('ENV PATH="/app/.venv/bin:$PATH"')
+
+        # 3. Check for requirements.txt (Use uv for speed)
+        elif (context_path / "requirements.txt").exists():
+            print("📦 Detected requirements.txt")
+            lines.append("COPY requirements.txt .")
+            lines.append("RUN uv venv && uv pip install -r requirements.txt")
+
+        # 4. Check for dynamic requirements list (Use uv for speed)
+        elif config.requirements:
+            print("📦 Detected dynamic requirements list")
+            reqs_str = " ".join([f'"{r}"' for r in config.requirements])
+            lines.append(f"RUN uv venv && uv pip install {reqs_str}")
+
+        # Copy source code
+        lines.append("COPY . .")
+
+        # Install project itself if needed (for uv/poetry)
+        if (context_path / "uv.lock").exists():
+            lines.append("RUN uv sync --frozen")
+        elif (context_path / "poetry.lock").exists():
+            lines.append("RUN poetry install --no-interaction --no-ansi")
+
+        # Env vars
+        for k, v in config.env_vars.items():
+            lines.append(f"ENV {k}={v}")
+
+        return "\n".join(lines)