flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/cli/stack_cli.py

@@ -98,23 +98,28 @@ def stack() -> None:
 @click.option("--config", "-c", help="Path to flowyml.yaml")
 def list_stacks(config: str | None) -> None:
     """List all configured stacks."""
-    from flowyml.utils.stack_config import load_config
+    from flowyml.plugins.stack_config import get_stack_manager
+    from flowyml.plugins.config import get_config
 
-    loader = load_config(config)
-    stacks = loader.list_stacks()
+    # Initialize config if path provided
+    if config:
+        get_config(config)
+
+    manager = get_stack_manager()
+    stacks = manager.list_stacks()
 
     if not stacks:
-        click.echo("No stacks configured. Create a flowyml.yaml file.")
+        click.echo("No stacks configured. Create a flowyml.yaml file with 'stacks:' section.")
         return
 
-    default = loader.get_default_stack()
+    active = manager.active_stack_name
 
-    click.echo("\nConfigured stacks:")
+    click.echo("\n📦 Configured stacks:")
     for stack_name in stacks:
-        marker = " (default)" if stack_name == default else ""
-        config_data = loader.get_stack_config(stack_name)
-        stack_type = config_data.get("type", "unknown")
-        click.echo(f"  • {stack_name}{marker} [{stack_type}]")
+        marker = " ✓ (active)" if stack_name == active else ""
+        stack = manager.get_stack(stack_name)
+        orch_type = stack.orchestrator.get("type", "local") if stack and stack.orchestrator else "local"
+        click.echo(f"  • {stack_name}{marker} [orchestrator: {orch_type}]")
     click.echo()
 
 
@@ -123,35 +128,143 @@ def list_stacks(config: str | None) -> None:
 @click.option("--config", "-c", help="Path to flowyml.yaml")
 def show_stack(stack_name: str, config: str | None) -> None:
     """Show detailed stack configuration."""
-    from flowyml.utils.stack_config import load_config
-    import yaml
+    from flowyml.plugins.stack_config import get_stack_manager
+    from flowyml.plugins.config import get_config
 
-    loader = load_config(config)
-    stack_config = loader.get_stack_config(stack_name)
+    # Initialize config if path provided
+    if config:
+        get_config(config)
 
-    if not stack_config:
+    manager = get_stack_manager()
+    stack = manager.get_stack(stack_name)
+
+    if not stack:
         click.echo(f"Stack '{stack_name}' not found", err=True)
+        available = manager.list_stacks()
+        if available:
+            click.echo(f"Available stacks: {', '.join(available)}")
         sys.exit(1)
 
-    click.echo(f"\nStack: {stack_name}")
-    click.echo(yaml.dump(stack_config, default_flow_style=False))
+    is_active = stack_name == manager.active_stack_name
+    status = " (active)" if is_active else ""
+
+    click.echo(f"\n📦 Stack: {stack_name}{status}")
+    click.echo("─" * 40)
+
+    # Show components
+    if stack.orchestrator:
+        click.echo(f"\n🎯 Orchestrator: {stack.orchestrator.get('type', 'unknown')}")
+        for k, v in stack.orchestrator.items():
+            if k != "type":
+                click.echo(f"   {k}: {v}")
+
+    if stack.artifact_store:
+        click.echo(f"\n💾 Artifact Store: {stack.artifact_store.get('type', 'unknown')}")
+        for k, v in stack.artifact_store.items():
+            if k != "type":
+                click.echo(f"   {k}: {v}")
+
+    if stack.experiment_tracker:
+        click.echo(f"\n📊 Experiment Tracker: {stack.experiment_tracker.get('type', 'unknown')}")
+
+    if stack.model_registry:
+        click.echo(f"\n📝 Model Registry: {stack.model_registry.get('type', 'unknown')}")
+
+    if stack.model_deployer:
+        click.echo(f"\n🚀 Model Deployer: {stack.model_deployer.get('type', 'unknown')}")
+
+    if stack.container_registry:
+        click.echo(f"\n🐳 Container Registry: {stack.container_registry.get('type', 'unknown')}")
+
+    if stack.artifact_routing:
+        click.echo("\n📍 Artifact Routing:")
+        for type_name, rule in stack.artifact_routing.rules.items():
+            click.echo(f"   {type_name}: store={rule.store}, register={rule.register}")
+
+    click.echo()
 
 
 @stack.command("set-default")
 @click.argument("stack_name")
 @click.option("--config", "-c", help="Path to flowyml.yaml")
-def set_default_stack(stack_name: str, config: str | None) -> None:
-    """Set the default stack."""
-    from flowyml.stacks.registry import get_registry
+def set_active_stack(stack_name: str, config: str | None) -> None:
+    """Set the active stack (alias for switch)."""
+    switch_stack_impl(stack_name, config)
 
-    registry = get_registry()
 
-    if stack_name not in registry.list_stacks():
-        click.echo(f"Stack '{stack_name}' not found", err=True)
+@stack.command("switch")
+@click.argument("stack_name")
+@click.option("--config", "-c", help="Path to flowyml.yaml")
+@click.option("--validate/--no-validate", default=True, help="Validate stack configuration after switching")
+def switch_stack(stack_name: str, config: str | None, validate: bool) -> None:
+    """Switch to a different stack."""
+    switch_stack_impl(stack_name, config, validate)
+
+
+def switch_stack_impl(stack_name: str, config: str | None, validate: bool = False) -> None:
+    from flowyml.plugins.stack_config import get_stack_manager
+    from flowyml.plugins.config import get_config
+
+    from rich.console import Console
+
+    console = Console()
+
+    # Initialize config if path provided
+    if config:
+        get_config(config)
+
+    manager = get_stack_manager()
+
+    if manager.set_active_stack(stack_name):
+        console.print(f"[bold green]✅ Active stack set to '{stack_name}'[/bold green]")
+
+        if validate:
+            stack = manager.get_stack(stack_name)
+            console.print(f"🔍 Validating stack '{stack_name}' environment...")
+
+            # Check for remote requirements
+            if stack.orchestrator and stack.orchestrator.get("type") != "local":
+                console.print("[yellow]Remote stack detected. Checking Docker configuration...[/yellow]")
+                if not stack.container_registry:
+                    console.print(
+                        "[bold red]❌ Warning: Remote stack usually requires a Container Registry for automatic builds.[/bold red]",
+                    )
+                    console.print(
+                        "   Please configure 'container_registry' in your stack or ensure you provide pre-built image URIs.",
+                    )
+    else:
+        console.print(f"[bold red]❌ Stack '{stack_name}' not found[/bold red]")
+        available = manager.list_stacks()
+        if available:
+            console.print(f"Available stacks: {', '.join(available)}")
+        sys.exit(1)
+
+
+@stack.command("register")
+@click.argument("stack_name")
+@click.option("--file", "-f", "config_file", required=True, help="Path to stack config file")
+def register_stack(stack_name: str, config_file: str) -> None:
+    """Register a new stack from a config file."""
+    import yaml
+    from flowyml.plugins.stack_config import get_stack_manager, StackConfig
+
+    config_path = Path(config_file)
+    if not config_path.exists():
+        click.echo(f"Config file not found: {config_file}", err=True)
         sys.exit(1)
 
-    registry.set_active_stack(stack_name)
-    click.echo(f"Set '{stack_name}' as active stack")
+    try:
+        with open(config_path) as f:
+            stack_data = yaml.safe_load(f)
+
+        stack_config = StackConfig.from_dict(stack_name, stack_data)
+        manager = get_stack_manager()
+        manager.register_stack(stack_name, stack_config)
+
+        click.echo(f"✅ Registered stack '{stack_name}' from {config_file}")
+    except Exception as e:
+        click.echo(f"❌ Error registering stack: {e}", err=True)
+        sys.exit(1)
 
 
 @cli.command()

flowyml/core/__init__.py

@@ -27,6 +27,15 @@ from flowyml.core.resources import (
     NodeAffinity,
     resources,
 )
+from flowyml.core.types import (
+    Artifact,
+    Model,
+    Dataset,
+    Metrics,
+    Parameters,
+    is_artifact_type,
+    get_artifact_type_name,
+)
 
 __all__ = [
     # Context
@@ -65,4 +74,12 @@ __all__ = [
     "GPUConfig",
     "NodeAffinity",
     "resources",
+    # Artifact Types
+    "Artifact",
+    "Model",
+    "Dataset",
+    "Metrics",
+    "Parameters",
+    "is_artifact_type",
+    "get_artifact_type_name",
 ]

flowyml/core/executor.py

@@ -12,6 +12,7 @@ import ctypes
 import requests
 import os
 import inspect
+import psutil
 
 
 class StopExecutionError(Exception):
@@ -87,10 +88,16 @@ class MonitorThread(threading.Thread):
             # Fallback to environment variable or default
             self.api_url = os.getenv("FLOWYML_SERVER_URL", "http://localhost:8080")
 
-    def stop(self):
+    def stop(self, error: str | None = None):
+        """Stop the monitor thread.
+
+        Args:
+            error: Optional error message to send as final log entry
+        """
+        self._final_error = error
         self._stop_event.set()
 
-    def _flush_logs(self):
+    def _flush_logs(self, level: str = "INFO"):
         """Send captured logs to the server."""
         if not self.log_capture:
             return
@@ -105,7 +112,20 @@ class MonitorThread(threading.Thread):
                 f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/logs",
                 json={
                     "content": content,
-                    "level": "INFO",
+                    "level": level,
+                    "timestamp": datetime.now().isoformat(),
+                },
+                timeout=2,
+            )
+
+    def _send_error(self, error: str):
+        """Send error message to the server."""
+        with contextlib.suppress(Exception):
+            requests.post(
+                f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/logs",
+                json={
+                    "content": f"ERROR: {error}",
+                    "level": "ERROR",
                     "timestamp": datetime.now().isoformat(),
                 },
                 timeout=2,
@@ -114,10 +134,25 @@ class MonitorThread(threading.Thread):
     def run(self):
         while not self._stop_event.is_set():
             try:
-                # Send heartbeat
+                # Collect metrics
+                process = psutil.Process()
+                with process.oneshot():
+                    cpu_percent = process.cpu_percent(interval=None)
+                    memory_info = process.memory_info()
+                    memory_mb = memory_info.rss / 1024 / 1024
+
+                # Send heartbeat with metrics
                 response = requests.post(
                     f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/heartbeat",
-                    json={"step_name": self.step_name, "status": "running"},
+                    json={
+                        "step_name": self.step_name,
+                        "status": "running",
+                        "metrics": {
+                            "cpu_percent": cpu_percent,
+                            "memory_mb": memory_mb,
+                            "timestamp": datetime.now().isoformat(),
+                        },
+                    },
                     timeout=2,
                 )
                 if response.status_code == 200:
@@ -137,6 +172,10 @@ class MonitorThread(threading.Thread):
         # Final log flush
         self._flush_logs()
 
+        # Send error if there was one
+        if hasattr(self, "_final_error") and self._final_error:
+            self._send_error(self._final_error)
+
 
 # Keep HeartbeatThread as an alias for backwards compatibility
 HeartbeatThread = MonitorThread
@@ -171,6 +210,10 @@ class Executor:
         inputs: dict[str, Any],
         context_params: dict[str, Any],
         cache_store: Any | None = None,
+        artifact_store: Any | None = None,
+        run_id: str | None = None,
+        project_name: str = "default",
+        all_outputs: dict[str, Any] | None = None,
     ) -> ExecutionResult:
         """Execute a single step.
 
@@ -179,6 +222,10 @@ class Executor:
             inputs: Input data for the step
             context_params: Parameters from context
             cache_store: Cache store for caching
+            artifact_store: Artifact store for logging results
+            run_id: Unique ID for this pipeline run
+            project_name: Name of the project
+            all_outputs: Collection of all step outputs for conditional evaluation
 
         Returns:
             ExecutionResult with output or error
@@ -226,21 +273,43 @@ class LocalExecutor(Executor):
         artifact_store: Any | None = None,
         run_id: str | None = None,
         project_name: str = "default",
+        all_outputs: dict[str, Any] | None = None,
     ) -> ExecutionResult:
-        """Execute step locally with retry, caching, and materialization."""
+        """Execute step locally with retry, caching, and materialization.
+
+        Args:
+            step: Step to execute
+            inputs: Input data for the step
+            context_params: Parameters from context
+            cache_store: Cache store for caching
+            artifact_store: Artifact store for logging results
+            run_id: Unique ID for this pipeline run
+            project_name: Name of the project
+            all_outputs: Collection of all step outputs for conditional evaluation
+
+        Returns:
+            ExecutionResult with output or error
+        """
         start_time = time.time()
         retries = 0
 
         # Check condition
         if step.condition:
             try:
-                # We pass inputs and context params to condition if it accepts them
-                # For simplicity, let's try to inspect the condition function
-                # or just pass what we can.
-                # A simple approach: pass nothing if it takes no args, or kwargs if it does.
-                # But inspect is safer.
+                # Prepare kwargs for condition: inputs + context_params + all_outputs
                 sig = inspect.signature(step.condition)
-                kwargs = {**inputs, **context_params}
+                kwargs = {**context_params}
+
+                # Add all outputs so far (paths like 'data/processed')
+                if all_outputs:
+                    kwargs.update(all_outputs)
+                    # Also flatten dict outputs to allow access to keys like 'quality_score'
+                    for val in all_outputs.values():
+                        if isinstance(val, dict):
+                            kwargs.update({k: v for k, v in val.items() if k not in kwargs})
+
+                # Add direct inputs (might override all_outputs if paths match)
+                kwargs.update(inputs)
 
                 # Filter kwargs to only what condition accepts
                 cond_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters}
@@ -314,7 +383,12 @@ class LocalExecutor(Executor):
                         )
                         monitor_thread.start()
 
-                    result = step.func(**kwargs)
+                    # Filter kwargs to only what the function accepts
+                    func_sig = inspect.signature(step.func)
+                    # Handle *args/**kwargs if needed, but for now strict matching is safer for steps
+                    filtered_kwargs = {k: v for k, v in kwargs.items() if k in func_sig.parameters}
+
+                    result = step.func(**filtered_kwargs)
                 except StopExecution:
                     duration = time.time() - start_time
                     return ExecutionResult(
@@ -335,22 +409,63 @@ class LocalExecutor(Executor):
 
                         sys.stderr = original_stderr
 
-                    # Stop monitor thread
-                    if monitor_thread:
+                    # Stop monitor thread (only if not already stopped in exception handler)
+                    if monitor_thread and not monitor_thread._stop_event.is_set():
                         monitor_thread.stop()
                         monitor_thread.join()
 
                 # Materialize output if artifact store is available
+                # Only upload if the result is an Asset with upload=True
                 artifact_uri = None
                 if artifact_store and result is not None and run_id:
-                    with contextlib.suppress(Exception):
-                        artifact_uri = artifact_store.materialize(
-                            obj=result,
-                            name="output",  # Default name for single output
-                            run_id=run_id,
+                    # Check if result is an Asset and respects upload flag
+                    should_upload = True
+                    try:
+                        from flowyml.assets.base import Asset
+
+                        if isinstance(result, Asset):
+                            should_upload = getattr(result, "upload", False)
+                    except ImportError:
+                        pass
+
+                    if should_upload:
+                        with contextlib.suppress(Exception):
+                            artifact_uri = artifact_store.materialize(
+                                obj=result,
+                                name="output",  # Default name for single output
+                                run_id=run_id,
+                                step_name=step.name,
+                                project_name=project_name,
+                            )
+
+                # Type-based artifact routing
+                routing_result = None
+                try:
+                    from flowyml.core.routing import route_artifact, should_route
+
+                    if should_route(result):
+                        # Get return type annotation if available
+                        return_type = None
+                        try:
+                            from flowyml.core.routing import get_step_return_type
+
+                            return_type = get_step_return_type(step.func)
+                        except Exception:
+                            pass
+
+                        routing_result = route_artifact(
+                            output=result,
                             step_name=step.name,
+                            run_id=run_id or "local",
+                            return_type=return_type,
                             project_name=project_name,
                         )
+                        if routing_result and routing_result.store_uri:
+                            artifact_uri = routing_result.store_uri
+                except ImportError:
+                    pass  # Routing module not available
+                except Exception:
+                    pass  # Routing failed, continue with normal flow
 
                 # Cache result
                 if cache_store and step.cache:
@@ -374,6 +489,7 @@ class LocalExecutor(Executor):
 
             except Exception as e:
                 last_error = str(e)
+                error_traceback = traceback.format_exc()
                 retries += 1
 
                 if attempt < max_retries:
@@ -382,12 +498,17 @@ class LocalExecutor(Executor):
                     time.sleep(wait_time)
                     continue
 
-                # All retries exhausted
+                # All retries exhausted - send error to logs
+                if monitor_thread:
+                    monitor_thread.stop(error=f"{last_error}\n{error_traceback}")
+                    monitor_thread.join()
+                    monitor_thread = None  # Prevent double-stop in finally
+
                 duration = time.time() - start_time
                 return ExecutionResult(
                     step_name=step.name,
                     success=False,
-                    error=f"{last_error}\n{traceback.format_exc()}",
+                    error=f"{last_error}\n{error_traceback}",
                     duration_seconds=duration,
                     retries=retries,
                 )
@@ -438,11 +559,41 @@ class LocalExecutor(Executor):
             # Find the step object
             step = next(s for s in step_group.steps if s.name == step_name)
 
-            # Prepare inputs for this step
+            # Prepare inputs for this step - map input names to function parameters
             step_inputs = {}
-            for input_name in step.inputs:
-                if input_name in step_outputs:
-                    step_inputs[input_name] = step_outputs[input_name]
+
+            # Get function signature to properly map inputs to parameters
+            sig = inspect.signature(step.func)
+            params = list(sig.parameters.values())
+            # Filter out self/cls
+            params = [p for p in params if p.name not in ("self", "cls")]
+            assigned_params = set()
+
+            if step.inputs:
+                for i, input_name in enumerate(step.inputs):
+                    if input_name not in step_outputs:
+                        continue
+
+                    val = step_outputs[input_name]
+
+                    # Check if input name matches a parameter directly
+                    param_match = next((p for p in params if p.name == input_name), None)
+
+                    if param_match:
+                        step_inputs[param_match.name] = val
+                        assigned_params.add(param_match.name)
+                    elif i < len(params):
+                        # Positional fallback - use the parameter at the same position
+                        target_param = params[i]
+                        if target_param.name not in assigned_params:
+                            step_inputs[target_param.name] = val
+                            assigned_params.add(target_param.name)
+
+            # Auto-map parameters from available outputs by name
+            for param in params:
+                if param.name in step_outputs and param.name not in step_inputs:
+                    step_inputs[param.name] = step_outputs[param.name]
+                    assigned_params.add(param.name)
 
             # Inject context parameters for this specific step
             if context is not None:
@@ -515,33 +666,76 @@ class DistributedExecutor(Executor):
         inputs: dict[str, Any],
         context_params: dict[str, Any],
         cache_store: Any | None = None,
+        artifact_store: Any | None = None,
+        run_id: str | None = None,
+        project_name: str = "default",
+        all_outputs: dict[str, Any] | None = None,
     ) -> ExecutionResult:
-        """Execute step in distributed manner."""
+        """Execute step in distributed manner.
+
+        Args:
+            step: Step to execute
+            inputs: Input data for the step
+            context_params: Parameters from context
+            cache_store: Cache store for caching
+            artifact_store: Artifact store for logging results
+            run_id: Unique ID for this pipeline run
+            project_name: Name of the project
+            all_outputs: Collection of all step outputs for conditional evaluation
+
+        Returns:
+            ExecutionResult with output or error
+        """
         # Placeholder - would use Ray, Dask, or similar
         # For now, fall back to local execution
         local_executor = LocalExecutor()
-        return local_executor.execute_step(step, inputs, context_params, cache_store)
+        return local_executor.execute_step(
+            step,
+            inputs,
+            context_params,
+            cache_store,
+            artifact_store,
+            run_id,
+            project_name,
+            all_outputs,
+        )
 
     def execute_step_group(
         self,
         step_group,  # StepGroup
         inputs: dict[str, Any],
-        context_params: dict[str, Any],
+        context: Any | None = None,  # Context object for per-step injection
+        context_params: dict[str, Any] | None = None,  # Deprecated: use context instead
         cache_store: Any | None = None,
         artifact_store: Any | None = None,
         run_id: str | None = None,
         project_name: str = "default",
     ) -> list[ExecutionResult]:
-        """Execute step group in distributed manner."""
+        """Execute step group in distributed manner.
+
+        Args:
+            step_group: StepGroup to execute
+            inputs: Input data available to the group
+            context: Context object for per-step parameter injection (preferred)
+            context_params: Parameters from context (deprecated, use context instead)
+            cache_store: Cache store for caching
+            artifact_store: Artifact store for materialization
+            run_id: Run identifier
+            project_name: Project name
+
+        Returns:
+            List of ExecutionResult (one per step)
+        """
         # Placeholder - in real implementation, would send entire group to remote worker
         # For now, fall back to local execution
         local_executor = LocalExecutor()
         return local_executor.execute_step_group(
-            step_group,
-            inputs,
-            context_params,
-            cache_store,
-            artifact_store,
-            run_id,
-            project_name,
+            step_group=step_group,
+            inputs=inputs,
+            context=context,
+            context_params=context_params,
+            cache_store=cache_store,
+            artifact_store=artifact_store,
+            run_id=run_id,
+            project_name=project_name,
         )